<jupyter_start><jupyter_text>Pré-Processamento dos DadosCiência e Visualização de Dados em Saúde (2021.1)UNICAMP - Universidade Estadual de Campinas<NAME>, 190943 (Saúde/FCM) \\<NAME>, 093801 (Computação/FEEC) \\<NAME>, 117119 (Computação/FEEC) \\<NAME>, 211518 (Computação/IC)---<jupyter_code>from google.colab import drive
drive.mount('/content/drive')
root_1 = 'drive/Shareddrives/MO286 - Visualização de Dados/data_csv/raw_sheets/'
!ls 'drive/Shareddrives/MO286 - Visualização de Dados/data_csv/raw_sheets/'
root = 'drive/Shareddrives/MO286 - Visualização de Dados/data_csv/'
!ls 'drive/Shareddrives/MO286 - Visualização de Dados/data_csv/'
root_imgs = 'drive/Shareddrives/MO286 - Visualização de Dados/data_images/CASOS_CHC_FULL/'

# !ls 'drive/Shareddrives/MO286 - Visualização de Dados/data_images/CASOS_CHC_FULL'
!pip install unidecode
# Importação de pacotes
import pandas as pd
import numpy as np
import sklearn
import matplotlib.pyplot as plt
import seaborn as sns
import unidecode
import glob
import os
# Removendo colunas vazias
def remove_null_cols(dataset):
  dataset.drop([c for c in dataset if c.startswith('Unnamed')], axis=1, inplace=True)

# Abrir todos os dados
s1 = pd.read_csv(root_1+'s1.csv')
s2 = pd.read_csv(root_1+'s2.csv')
s3 = pd.read_csv(root_1+'s3.csv')
s4 = pd.read_csv(root_1+'s4.csv')
s5 = pd.read_csv(root_1+'s5.csv')
s6 = pd.read_csv(root_1+'s6.csv')
s7 = pd.read_csv(root_1+'s7.csv')
s8 = pd.read_csv(root_1+'s8.csv')
s9 = pd.read_csv(root_1+'s9.csv')

all_datasets = [s1, s2, s3, s4, s5, s6, s7, s8, s9]

for dataset in all_datasets:
  remove_null_cols(dataset)
# Concatenando todos os dados para visualização de todas as features
dataset = pd.concat([s1, s2], axis=0, ignore_index=True)
dataset = pd.concat([dataset, s3], axis=0, ignore_index=True)
dataset = pd.concat([dataset, s4], axis=0, ignore_index=True)
dataset = pd.concat([dataset, s5], axis=0, ignore_index=True)
dataset = pd.concat([dataset, s6], axis=0, ignore_index=True)
dataset = pd.concat([dataset, s7], axis=0, ignore_index=True)
dataset = pd.concat([dataset, s8], axis=0, ignore_index=True)
dataset = pd.concat([dataset, s9], axis=0, ignore_index=True)
print('Quantidade total de features: {}'.format(len(list(dataset))))
# Abrindo mapeamento de colunas
new_col_names = pd.read_csv(root+'columns.csv')
new_col_names.drop_duplicates(subset=['old'], inplace=True)
print('Number of unique columns: ', new_col_names.shape)
# Alterando nome das colunas para facilitar acesso
def change_column_names(data, ref):
  dataset = data.copy()
  final_col =[]
  for old_c in dataset.columns:
    new_c = ref.new[ref.old==old_c].values
    if new_c:
      final_col.append(new_c[0])

  return final_col
def remove_accents(string):
  return unidecode.unidecode(string)
dataset.columns = change_column_names(dataset, new_col_names)

# Colocando nomes em minúsculo
dataset.paciente = dataset.paciente.str.lower()

print('Quantidade de pacientes: {}'.format(len(np.unique(dataset.paciente.dropna()))))
# Pacientes <-> anonimização geral

dataset = dataset.dropna(subset=['paciente'])
dataset.paciente = dataset.paciente.str.lower()
dataset.paciente = dataset.paciente.str.strip()
dataset.paciente = dataset.paciente.apply(remove_accents)

c = dataset.paciente.astype('category')

d = dict(enumerate(c.cat.categories))
mapping_names = pd.DataFrame(list(d.items()), columns=['key', 'name'])
# mapping_names.to_csv(root+'mapeamento_nomes.csv')
mapping_names = pd.read_csv(root+'mapeamento_nomes.csv')
def name_to_cat(name):
  return int(mapping_names.key[mapping_names.name == name])
cnt=0
for dataset in all_datasets:
  cnt+=1
  dataset.columns = change_column_names(dataset, new_col_names)

  if 'paciente' not in dataset.columns:

    print("s" + str(cnt) + ".csv: " + "não contém coluna paciente")
    continue

  else:

    try:
      
      dataset.dropna(subset=['paciente'], inplace=True)

      dataset.paciente = dataset.paciente.str.lower()
      dataset.paciente = dataset.paciente.str.strip()
      dataset.paciente = dataset.paciente.apply(remove_accents)
      dataset['paciente_id'] = dataset.paciente.apply(name_to_cat)

    except AttributeError as err:
      print("s" + str(cnt) + ".csv: " + str(err))
      continue<jupyter_output>s7.csv: não contém coluna paciente
s9.csv: não contém coluna paciente<jupyter_text>Anonimizar os dadosSerão removidas as colunas de identificação.<jupyter_code>for dataset in all_datasets:
  dataset.drop(['paciente', 'hc','data_tc','data_tx', 'data_tx_cx','data_nasc'], axis='columns', inplace=True, errors='ignore')<jupyter_output><empty_output><jupyter_text>Anonimizando pastas de imagens.<jupyter_code>def get_name_cat(name):

  name = name.lower().strip()
  name = remove_accents(name)
  name = name_to_cat(name)

  return name
# folders_path = glob.glob(root_imgs+'*')

# for folder_path in folders_path:

#   folder_name = os.path.basename(folder_path)
#   folder = os.path.dirname(folder_path)

#   p_name = get_name_cat(folder_name)

#   print(folder, ' || ', folder_name, ' || ', p_name)
#   break<jupyter_output><empty_output><jupyter_text>Colunas restantes em cada dataset<jupyter_code>s1.head()
s2.head()
s3.head()
s4.head()
s5.head()
s6.head()
s7.head()
s8.head()
s9.head()<jupyter_output><empty_output><jupyter_text>Analisar dados faltantes<jupyter_code># Dados faltantes em todas as linhas da coluna
cnt = 0
print('Colunas onde todos os dados estão faltando')
for dataset in all_datasets:
  cnt += 1
  missing_data = dataset.columns[dataset.isna().all()]
  if not missing_data.empty:
    print('s' + str(cnt) + '.csv:', *missing_data, sep='\n- ')
# Remover colunas onde todos os valores são nulos
for dataset in all_datasets:
  dataset.dropna(axis=1, how='all', inplace=True)<jupyter_output><empty_output><jupyter_text>Mapeando valores categóricos<jupyter_code># Alterando dados de , para .

# dataset.tam_nod_tc_cm = dataset.tam_nod_tc_cm.str.replace(',','.').astype(float)
# dataset.par_portal = dataset.par_portal.str.replace(',','.').astype(float)
# dataset.par_equi = dataset.par_equi.str.replace(',','.').astype(float)
# dataset.ac_arterial = dataset.ac_arterial.str.replace(',','.').astype(float)
# dataset.ac_portal = dataset.ac_portal.str.replace(',','.').astype(float)
# dataset.ac_equi = dataset.ac_equi.str.replace(',','.').astype(float)
# dataset.rwr_portal = dataset.rwr_portal.str.replace(',','.').astype(float)
# dataset.rwr_equi = dataset.rwr_equi.str.replace(',','.').astype(float)
# dataset.aef = dataset.aef.str.replace(',','.').astype(float)
cnt = 0
for dataset in all_datasets:
  cnt += 1
  try:
    dataset.sexo = dataset.sexo.str.replace('M','0').replace('F', '1').astype(int)
    dataset.invasao_microvascular = dataset.invasao_microvascular.str.replace('ausente','0').replace('presente', '1').astype(int)
  except AttributeError as err:
    print("s" + str(cnt) + ".csv: " + str(err))
    continue<jupyter_output>s1.csv: 'DataFrame' object has no attribute 'sexo'
s2.csv: 'DataFrame' object has no attribute 'sexo'
s3.csv: 'DataFrame' object has no attribute 'sexo'
s7.csv: 'DataFrame' object has no attribute 'sexo'
s8.csv: 'DataFrame' object has no attribute 'sexo'
s9.csv: 'DataFrame' object has no attribute 'sexo'<jupyter_text>Mapeamento dos valores do grau histológico do paciente*   1 $\implies 1$*   2 $\implies 2$*   3 $\implies 3$*   4 $\implies 4$*   1, 2 $\implies 5$*   1, 3 $\implies 6$*   1, 4 $\implies 7$*   2, 3 $\implies 8$*   2, 4 $\implies 9$*   3, 4 $\implies 10$*   sem grau $\implies 11$*   quimioembolização $\implies 12$*   não informado $\implies 44$*   nan $\implies 44$<jupyter_code>cnt = 0
for dataset in all_datasets:
  cnt += 1
  try:
    dataset.grau_histologico = dataset.grau_histologico.replace('sem grau','11').replace('não informado', '44').replace(np.nan, '44')
    dataset.grau_histologico = dataset.grau_histologico.replace('1 e 2', '5').replace('1 e 3', '6').replace('1 e 4', '7')
    dataset.grau_histologico = dataset.grau_histologico.replace('2 e 3', '8').replace('2 e 4', '9').replace('3 e 4', '10')
    dataset.grau_histologico = dataset.grau_histologico.replace('quimioembolização','12').astype(int)
  except AttributeError as err:
    print("s" + str(cnt) + ".csv: " + str(err))
    continue
  except ValueError as err:
    print("s" + str(cnt) + ".csv: " + str(err))
    continue<jupyter_output>s1.csv: 'DataFrame' object has no attribute 'grau_histologico'
s2.csv: invalid literal for int() with base 10: 'não se aplica'
s3.csv: 'DataFrame' object has no attribute 'grau_histologico'
s7.csv: 'DataFrame' object has no attribute 'grau_histologico'
s8.csv: 'DataFrame' object has no attribute 'grau_histologico'
s9.csv: 'DataFrame' object has no attribute 'grau_histologico'<jupyter_text>Mapeamento da causa hepatopatia da lesão*   CRIPTO $\implies 1$*   OH $\implies 2$*   OH + VHB $\implies 3$*   OH + VHC $\implies 4$*   VHB $\implies 5$*   VHC $\implies 6$*   VHC + OH $\implies 7$*   VHC + VHB $\implies 8$*   NaN $\implies 44$<jupyter_code>cnt = 0
for dataset in all_datasets:
  cnt += 1
  try:
    dataset.causa_hepatopatia = dataset.causa_hepatopatia.replace(np.nan, '44')
    dataset.causa_hepatopatia = dataset.causa_hepatopatia.replace('CRIPTO', '1').replace('OH', '2')
    dataset.causa_hepatopatia = dataset.causa_hepatopatia.replace('OH + VHB', '3').replace('OH + VHC', '4')
    dataset.causa_hepatopatia = dataset.causa_hepatopatia.replace('VHB', '5').replace('VHC', '6')
    dataset.causa_hepatopatia = dataset.causa_hepatopatia.replace('VHC + OH', '7').replace('VHC + VHB', '8').astype(int)
  except AttributeError as err:
    print("s" + str(cnt) + ".csv: " + str(err))
    continue<jupyter_output>s1.csv: 'DataFrame' object has no attribute 'causa_hepatopatia'
s2.csv: 'DataFrame' object has no attribute 'causa_hepatopatia'
s3.csv: 'DataFrame' object has no attribute 'causa_hepatopatia'
s7.csv: 'DataFrame' object has no attribute 'causa_hepatopatia'
s8.csv: 'DataFrame' object has no attribute 'causa_hepatopatia'
s9.csv: 'DataFrame' object has no attribute 'causa_hepatopatia'<jupyter_text>Salvando os Dados Pré-processados<jupyter_code>s1.to_csv(root+'s1.csv', index=False)
s2.to_csv(root+'s2.csv', index=False)
s3.to_csv(root+'s3.csv', index=False)
s4.to_csv(root+'s4.csv', index=False)
s5.to_csv(root+'s5.csv', index=False)
s6.to_csv(root+'s6.csv', index=False)
s7.to_csv(root+'s7.csv', index=False)
s8.to_csv(root+'s8.csv', index=False)
s9.to_csv(root+'s9.csv', index=False)<jupyter_output><empty_output><jupyter_text>Renoveando diretórios<jupyter_code>import os
import pandas as pd
import numpy as np
from glob import glob
import unidecode
root_images =  'drive/Shareddrives/MO286 - Visualização de Dados/data_images/CASOS_CHC_FULL/*'	
root_csv =  'drive/Shareddrives/MO286 - Visualização de Dados/data_csv/'
image_csv = 'drive/Shareddrives/MO286 - Visualização de Dados/data_csv/mapeamento_nomes.csv'		
dataset = pd.read_csv(root_csv+'s5.csv')
mapping_names = pd.read_csv(image_csv)
def remove_accents(string):
  return unidecode.unidecode(string)
def name_to_cat(name):
  try:
    return int(mapping_names.key[mapping_names.name == name])
  except:
    return 'exit'
# Rename folders

name_backup = []

img_root = 'drive/Shareddrives/MO286 - Visualização de Dados/data_images/CASOS_CHC_FULL/'

img_folders = glob(root_images)

for folder_name in img_folders:

  path_ = os.path.dirname(folder_name)
  name_ = os.path.basename(folder_name)
  name = name_.strip().lower()
  name = remove_accents(name)

  name_to_code = name_to_cat(name)

  if name_to_code == 'exit':   
    continue

  name_backup.append(folder_name)

  os.rename(folder_name,  path_+'/'+str(name_to_code))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1488997345663156, 'fraction_numerical': 0.022433427519479406, 'mean_word_length': 5.4494754279403645, 'pattern_counts': {'":': 0, '<': 42, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '652160', 'license': 'MIT', 'n_tokens_mistral': 4668, 'n_tokens_neox': 4470, 'n_words': 897, 'path': 'notebooks/dataset_pre_processing.ipynb', 'repo_name': 'paulinog/2021.1-CienciaDeDadosEmSaude_IA368X-MO826A-CX002A', 'size': '11755'}