<jupyter_start><jupyter_text><jupyter_code># Load the Drive helper and mount
from google.colab import drive

# This will prompt for authorization.menos de 48hs de evolución
drive.mount('/content/drive')
path = "/content/drive/My Drive/NLP/GPT2/autonomous/"
path2 = "/content/drive/My Drive/NLP/summary/"
import pandas as pd

df = pd.read_csv (path + 'uma_18_01_2021.csv')
import pandas as pd
df2 = pd.read_json (path2 + 'speech_text.json')
df2.info()
df.incidente_id
df_3 = df.set_index('assignation_id').join(df2.set_index('id'))
df3_3 = df_3.drop_duplicates()
df_3['texto_edit'] = df_3.text.str.len()
df_3['texto_edit'] = df_3.texto_edit.fillna(0)
df_3['mr_epicrisis_edit'] = df_3.mr_epicrisis.str.len()
df_3['mr_epicrisis_edit'] = df_3.mr_epicrisis_edit.fillna(0)
df_3.mr_epicrisis_edit.sample(10)
df4 = df_3[['provider_cuit', 'mr_dt_cierre', 'mr_preds_pre_clasif', 'mr_reposo',
       'mr_epicrisis', 'mr_epicrisis_edit', 'mr_destino_final', 'mr_receta', 'texto_edit']]
dummy2 = pd.get_dummies(df4['mr_destino_final'], prefix='destino')
df4 = pd.concat([df4, dummy2], axis = 1)

#convierto a datime las variables temporales y genero la variable "duración_consulta_segundos"
df4['ts_cierre'] = pd.to_datetime(df4["mr_dt_cierre"], errors='coerce')
df4['ts_inicio'] = pd.to_datetime(df4["mr_preds_pre_clasif"], errors='coerce')
df4['tiempo_consulta'] = df4.ts_cierre - df4.ts_inicio
df4['duración_consulta_segundos'] = df4.tiempo_consulta.dt.total_seconds()
df5 = df4.drop_duplicates()
df5.columns
df6 = df5
pd.set_option('display.max_colwidth', -1)
df6.sample(10)
df6.to_csv(path2 + 'fraude_medicos.csv', index=False )
df6 = pd.read_csv (path2 + 'fraude_medicos.csv' )
df6.columns
df6_filter = df6[['provider_cuit', 'mr_reposo',
      'mr_epicrisis_edit', 'mr_receta',
       'texto_edit', 'destino_Anula el paciente',
       'destino_Anula por falla de conexión',
       'destino_En domicilio con instrucciones',
       'destino_En domicilio con monitoreo',
       'destino_En domicilio con seguimiento',
       'destino_Evaluación en amarillo', 'destino_Evaluación en rojo',
       'destino_Evaluación en verde VMD',
       'destino_Indico concurrir a guardia externa',
       'destino_Indico seguimiento por consultorio externo',
       'destino_Paciente ausente', 'destino_Traslado protocolo pandemia',
       'destino_USER CANCEL',
       'duración_consulta_segundos']]
df6_filter.sample(10)
df2 = df6_filter
#Remplazo los NaN de recetas y los que tuvieron emisión les asigno 1
df2.mr_receta = df2.mr_receta.fillna(0)
df2['receta'] = df2['mr_receta'].replace('[]', 0)
df2.receta[df2.receta != 0] = 1
#remplazo los NaN por O y si tiene asignado reposo asigno 1
df2.mr_reposo = df2.mr_reposo.fillna(0)
df2.mr_reposo[df2.mr_reposo != 0] = 1
#Genro la variable "piscofarmacos"
lista_recetas = df2.mr_receta.to_list()
import ast
lista_final = []
for x in lista_recetas:
  try:
    a = ast.literal_eval(x)
    lista_final.append (a)
  except:
    lista_final.append ([])

duplicado = []
for x in lista_final:
  if x != []:
    if x != 0:
      try:
        duplicados = []
        for y in x:
          duplicados.append (int(y['duplicado']))
      except:
        duplicados = []
    else:
      duplicados = []
  else:
    duplicados = []
  duplicado.append (duplicados)
  
psico = []
for x in duplicado:
 psico.append (sum (x))
 
df2 ['psicofarmacos'] = psico
df2.columns
#Genero la variable "consultas" con el número de consultas que realizó cada provider 
#consultas = pd.DataFrame(df2.provider_cuit.value_counts().reset_index())
#consultas.columns = ['provider_cuit', 'consultas']
#merged = pd.merge(df2, consultas, on='provider_cuit')
#renombro el dataset
df_restore = df2
df_restore.sample(10)
#extraigo las variables de duración de las consultas de cada paciente
df_tiempo = df_restore.groupby('provider_cuit').duración_consulta_segundos.agg(['mean', 'median'])
df_tiempo.columns = ['mean_tiempo', 'median_tiempo']
#Genero la variable "consultas" con el número de consultas que realizó cada provider 
consultas = pd.DataFrame(df2.provider_cuit.value_counts().reset_index())
consultas.columns = ['provider_cuit', 'consultas']
merged = pd.merge(consultas, df_tiempo, on='provider_cuit')
df_texto = df_restore.groupby('provider_cuit').texto_edit.agg(['mean', 'median'])
df_texto.columns = ['mean_texto', 'median_texto']
merged_1 = pd.merge(merged, df_texto, on='provider_cuit')
df_epicrisis = df_restore.groupby('provider_cuit').mr_epicrisis_edit.agg(['mean', 'median'])
df_epicrisis.columns = ['mean_epicrisis', 'median_epicrisis']
merged_2 = pd.merge(merged_1, df_epicrisis, on='provider_cuit')
# sumo los pedidos de psicofarmacos de cada paciente y lo agrego al dataset anterior
df_psico = df_restore.groupby('provider_cuit').psicofarmacos.agg(['sum'])
df_psico.columns = ['psicofarmaco']
merged_3 = pd.merge(merged_2, df_psico, on='provider_cuit')
#sumo las recetas de cada paciente 
df_recetas = df_restore.groupby('provider_cuit').receta.agg(['sum'])
df_recetas.columns = ['recetas']
merged_4 = pd.merge(merged_3, df_recetas, on='provider_cuit')
#sumo los pedidos de reposo y los agrego al dataframe anterior
df_reposo = df_restore.groupby('provider_cuit').mr_reposo.agg(['sum'])
df_reposo.columns = ['reposo']
merged_5 = pd.merge(merged_4, df_reposo, on='provider_cuit')
df_restore.columns
df_ausente = df_restore.groupby('provider_cuit')['destino_Paciente ausente'].agg(['sum'])
df_ausente.columns = ['ausente']
merged_6 = pd.merge(merged_5, df_ausente, on='provider_cuit')
df_conexion = df_restore.groupby('provider_cuit')['destino_Anula por falla de conexión'].agg(['sum'])
df_conexion.columns = ['conexion']
merged_7 = pd.merge(merged_6, df_conexion, on='provider_cuit')
def variable (colum, name, merge):
  df = 'df_' + name
  df = df_restore.groupby('provider_cuit')[str(colum)].agg(['sum'])
  df.columns = [str(name)]
  merged = pd.merge(merge, df, on='provider_cuit')
  return merged
merged_8 = variable ('destino_En domicilio con instrucciones', 'domicilio', merged_7)
merged_9 = variable ('destino_Evaluación en amarillo', 'amarillo', merged_8)
merged_10 = variable ('destino_Evaluación en rojo', 'rojo', merged_9)
merged_11 = variable ('destino_Evaluación en verde VMD', 'verde', merged_10)
fraude = merged_11.dropna()
fraude.head(10)
fraude.to_csv(path + 'fraude_dataset')
fraude
df_metrics = pd.read_csv (path + 'providers_metrics.csv')
df_metrics = df_metrics[['cuit', 'metrics_puntualidad', 'metrics_stars',
       'metrics_sum_puntualidad', 'metrics_sum_stars']]
df_metrics.columns = ['provider_cuit', 'metrics_puntualidad', 'metrics_stars',
       'metrics_sum_puntualidad', 'metrics_sum_stars']
fraude_merged = pd.merge(fraude, df_metrics, on='provider_cuit')
fraud_merged = fraude_merged.drop_duplicates()
fraud_merged2 = fraud_merged.dropna()
fraud_merged2.head()
fraud_merged2.to_csv(path + 'providers_dataset_final')
fraud_merged2 = pd.read_csv(path + 'providers_dataset_final', index_col=0)
fraud_merged2.info()
fraud_merged2.columns
df_fraude = fraud_merged2
features = ['consultas', 'mean_tiempo', 'median_tiempo',
       'mean_texto', 'median_texto', 'mean_epicrisis', 'median_epicrisis',
       'psicofarmaco', 'recetas', 'reposo', 'ausente', 'conexion', 'domicilio',
       'amarillo', 'rojo', 'verde', 'metrics_puntualidad', 'metrics_stars',
       'metrics_sum_puntualidad', 'metrics_sum_stars']
from sklearn.preprocessing import StandardScaler


x = df_fraude.loc[:, features].values
# Separating out the target
y = df_fraude.loc[:,['provider_cuit']].values
# Standardizing the features
x = StandardScaler().fit_transform(x)
from sklearn.decomposition import PCA
pca = PCA(n_components=2)
principalComponents = pca.fit_transform(x)
principalDf = pd.DataFrame(data = principalComponents
             , columns = ['principal component 1', 'principal component 2'])
principalDf.index=df_fraude.index
finalDf = pd.concat([principalDf, df_fraude[['provider_cuit']]], axis = 1)
finalDf


#GENERO PCA
from sklearn.metrics import mean_squared_error
pca = PCA(n_components=2)
pca.fit(x)
X_train = pca.transform(x)
X_dt = pca.inverse_transform(X_train)
rms = mean_squared_error(x, X_dt, squared=True)
#Armo un score 
score = []
for i in enumerate(x):
  rms = mean_squared_error(x[i[0]], X_dt[i[0]], squared=True)
  score.append (rms)
# genero un dict con score y dni
lista_dni = df_fraude.provider_cuit.to_list()
data = {'cuit': lista_dni, 'score': score}
df_score = pd.DataFrame.from_dict(data)
df_score.sort_values(by=['score'], ascending=False).head(15)
df_reindex = df_score.sort_values(by=['score'], ascending=False).reset_index(drop = True)
nombres = pd.read_csv (path + 'providers_name.csv')
df_nombres = nombres[['cuit', 'fullname']]
df_nombres.columns = ['provider_cuit', 'fullname']

final_merged = pd.merge(df_fraude, df_nombres, on='provider_cuit')
final_merged = final_merged.drop_duplicates()
final_merged = final_merged.dropna()
df_score.columns = ['provider_cuit', 'score']
final_merged_order = pd.merge(final_merged, df_score, on='provider_cuit')
provider_fraudolentos = [27953661514, 27954160756]
final_merged[final_merged.provider_cuit == '27953661514']


from sklearn.neighbors import NearestNeighbors
nbrs = NearestNeighbors(n_neighbors=2, algorithm='auto').fit(x)
distances, indices = nbrs.kneighbors(x)
X = StandardScaler().fit_transform(X)
from sklearn.cluster import DBSCAN
clustering = DBSCAN(eps=3, min_samples=3).fit(x)
c = clustering.labels_.tolist()
c.count(-1)
final_merged ['cluster'] = pd.Series(c) 
final_merged[final_merged.cluster == 1]
final_merged.median_tiempo.value_counts().head(30)
sabri = final_merged_order.sort_values(by=['score'], ascending=False)
sabri.columns
sabri_final = sabri[['score', 'cluster','fullname','provider_cuit', 'consultas', 'mean_tiempo', 'median_tiempo',
       'mean_texto', 'median_texto', 'mean_epicrisis', 'median_epicrisis',
       'psicofarmaco', 'recetas', 'reposo', 'ausente', 'conexion', 'domicilio',
       'amarillo', 'rojo', 'verde', 'metrics_puntualidad', 'metrics_stars',
       'metrics_sum_puntualidad', 'metrics_sum_stars']]
sabri = sabri_final.reset_index (drop=True)
sabri_final.head(20)
sabri_final.to_csv (path + 'fraudulent_providers.csv', index = False)
sabri_final.to_excel (path + 'fraudulent_providers.xlsx', index = False)
df3 = pd.read_csv (path + 'fraudulent_providers.csv')
df3.head(15)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13886741399304214, 'fraction_numerical': 0.018457672980286045, 'mean_word_length': 6.282899366643209, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '114554', 'license': 'MIT', 'n_tokens_mistral': 4166, 'n_tokens_neox': 3979, 'n_words': 741, 'path': 'fraud_identification_pca_dbscan.ipynb', 'repo_name': 'daquarti/AI', 'size': '10348'}