<jupyter_start><jupyter_text>Data Analysis<jupyter_code>"""
Visualize and analyze data.
    
@author: <NAME> - <EMAIL>
"""

%matplotlib inline
import numpy as np
import pandas as pd
import seaborn as sns
from ast import literal_eval
import matplotlib.pyplot as plt
plt.style.use(['ggplot','seaborn-paper'])<jupyter_output><empty_output><jupyter_text>Load Data<jupyter_code>path_low = './results/df_low.csv'
path_med = './results//df_med.csv'
path_high = './results/df_high.csv'
df_low = pd.read_csv(path_low, index_col= 'Unnamed: 0')
df_med = pd.read_csv(path_med, index_col= 'Unnamed: 0')
df_high = pd.read_csv(path_high, index_col= 'Unnamed: 0')
data = {'Low': df_low, 'Med': df_med, 'High': df_high}
df_data = pd.DataFrame()
df_data = pd.concat(data)
df_data
df_log = pd.DataFrame()
df_SVC = pd.DataFrame()
df_RF = pd.DataFrame()
df_MLP = pd.DataFrame()
deci=3
for i in (['LR','SVC', 'RF','MLP']):
    df_unido = pd.DataFrame()
    
    df_low = pd.DataFrame()
    df_low['Thresholds'] = literal_eval(df_data[i].Low.Thresholds)
    df_low['Precisions'] = literal_eval(df_data[i].Low.Precisions)
    df_low['Recalls'] = literal_eval(df_data[i].Low.Recalls)
    df_low['Resolution'] = 'Low' 
    df_low['False Positives Rate'] = pd.Series(literal_eval(df_data[i].Low.False_Positives_Rate))
    df_low['True Positive Rate'] = pd.Series(literal_eval(df_data[i].Low.True_Positive_Rate))
    df_low['Processing Time'] = df_data[i].Low.Processing_Time
    df_low['ROC AUC'] = df_data[i].Low.ROC_AUC
    df_low['PR AUC'] = df_data[i].Low.PR_AUC
    
    df_med = pd.DataFrame()
    df_med['Thresholds'] = literal_eval(df_data[i].Med.Thresholds)
    df_med['Precisions'] = literal_eval(df_data[i].Med.Precisions)
    df_med['Recalls'] = literal_eval(df_data[i].Med.Recalls)
    df_med['Resolution'] = 'Med'
    df_med['False Positives Rate'] = pd.Series(literal_eval(df_data[i].Med.False_Positives_Rate))
    df_med['True Positive Rate'] = pd.Series(literal_eval(df_data[i].Med.True_Positive_Rate)) 
    df_med['Processing Time'] = df_data[i].Med.Processing_Time   
    df_med['ROC AUC'] = df_data[i].Med.ROC_AUC
    df_med['PR AUC'] = df_data[i].Med.PR_AUC
    
    df_unido = df_low.append(df_med, ignore_index=True)
    
    df_high = pd.DataFrame()
    df_high['Thresholds'] = literal_eval(df_data[i].High.Thresholds)
    df_high['Precisions'] = literal_eval(df_data[i].High.Precisions)
    df_high['Recalls'] = literal_eval(df_data[i].High.Recalls)
    df_high['Resolution'] = 'High'
    df_high['False Positives Rate'] = pd.Series(literal_eval(df_data[i].High.False_Positives_Rate))
    df_high['True Positive Rate'] = pd.Series(literal_eval(df_data[i].High.True_Positive_Rate))   
    df_high['Processing Time'] = df_data[i].High.Processing_Time
    df_high['ROC AUC'] = df_data[i].High.ROC_AUC
    df_high['PR AUC'] = df_data[i].High.PR_AUC
    
    df_unido = df_unido.append(df_high, ignore_index=True)
    df_unido['Model'] = i 
    
    if i == 'LR':
        df_log = df_unido
    elif i == 'SVC':
         df_SVC = df_unido
    elif i == 'RF':
         df_RF = df_unido            
    elif i == 'MLP':
         df_MLP = df_unido
df_data_2 = df_log.append(df_SVC, ignore_index=True)
df_data_2 = df_data_2.append(df_RF, ignore_index=True)
df_data_2 = df_data_2.append(df_MLP, ignore_index=True)
df_data_2['PR AUC'] = df_data_2['PR AUC'].astype(float)
df_data_2['Processing Time'] = df_data_2['Processing Time'].astype(float)
df_data_2['ROC AUC'] = df_data_2['ROC AUC'].astype(float)
df_data_2_p = df_data_2[['Thresholds','Precisions','Resolution','Model', 'Processing Time']]
df_data_2_p.rename(columns={'Precisions':'Score'}, inplace=True)
df_data_2_p['Score Type'] = 'Precision'

df_data_2_r = df_data_2[['Thresholds','Recalls','Resolution','Model', 'Processing Time']]
df_data_2_r.rename(columns={'Recalls':'Score'}, inplace=True)
df_data_2_r['Score Type'] = 'Recall'

df_data_3 = df_data_2_p.append(df_data_2_r, ignore_index=True)<jupyter_output>/home/juanfe/anaconda3/lib/python3.7/site-packages/pandas/core/frame.py:4025: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  return super(DataFrame, self).rename(**kwargs)
/home/juanfe/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:3: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  This is separate from the ipykernel package so we can avoid doing imports until
/home/juanfe/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:7: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See t[...]<jupyter_text>Data Visualization Area Under the Precision-Recall Curve Heat Map<jupyter_code>size = (8, 7)
a4_dims = size
palette_3 = sns.color_palette("mako_r",3)
flatui = [ "#95a5a6", "#2ecc71", "#34495e", "#e74c3c"]
palette_4 = sns.color_palette(flatui)
#sns.palplot(sns.color_palette(flatui))
df_plot = df_data_2.pivot_table(index='Model', 
                                columns='Resolution', 
                                values='PR AUC')
reorderlist = ['MLP', 'RF', 'SVC', 'LR']
df_plot = df_plot.reindex(reorderlist)
custom_dict = {'Low':0,'Med':1,'High':2}
df_plot = pd.DataFrame(df_plot, columns=sorted(custom_dict, key=custom_dict.get))
ax = sns.heatmap(df_plot, annot=True, fmt='.3g', cmap="YlGnBu", 
                             cbar_kws={'label': 'Area Under the Precision-Recall Curve'})
ax.set(xlabel='Resolution', ylabel='Model')
ax.figure.set_size_inches(size)
plt.savefig('./results/graphics/HM_PR.pdf', format='pdf', bbox_inches='tight')<jupyter_output><empty_output><jupyter_text>Processing Time Training Heat Map<jupyter_code>df_plot = df_data_2.pivot_table(index='Model', 
                        columns='Resolution', 
                        values='Processing Time')
reorderlist = ['MLP', 'RF', 'SVC', 'LR']
df_plot = df_plot.reindex(reorderlist)
custom_dict = {'Low':0,'Med':1,'High':2}
df_plot = pd.DataFrame(df_plot, columns=sorted(custom_dict, key=custom_dict.get))
ax = sns.heatmap(df_plot, annot=True, fmt='.3g', cmap="YlGnBu", 
                 cbar_kws={'label': 'Processing Time Training ($s$)'})
ax.set(xlabel='Resolution', ylabel='Model')
ax.figure.set_size_inches(size)
plt.savefig('./results/graphics/HM_PT.pdf', format='pdf', bbox_inches='tight')<jupyter_output><empty_output><jupyter_text>Thresholds VS Score Logistic Regression<jupyter_code>df_plot = df_data_3[df_data_3['Model'].isin(['LR'])] 
fig, ax = plt.subplots(figsize=a4_dims)
ax = sns.lineplot(x="Thresholds", y="Score", hue="Resolution", style="Score Type",
                  palette=palette_3, legend="full", data=df_plot)
#plt.legend(loc='center left', bbox_to_anchor=(1.0, 0.5))
ax.set_xlim(-2.5,3.5)
plt.savefig('./results/graphics/ST_LR.pdf', format='pdf', bbox_inches='tight')
plt.show()<jupyter_output><empty_output><jupyter_text>Support Vector Machine<jupyter_code>df_plot = df_data_3[df_data_3['Model'].isin(['SVC'])]
fig, ax = plt.subplots(figsize=a4_dims)
ax = sns.lineplot(x="Thresholds", y="Score", hue="Resolution", style="Score Type", palette=palette_3, 
                  legend="full", data=df_plot)
ax.set_xlim(-1,4)
#plt.legend(loc='center left', bbox_to_anchor=(1.0, 0.5))
plt.savefig('./results/graphics/ST_SVC.pdf', format='pdf', bbox_inches='tight')
plt.show()<jupyter_output><empty_output><jupyter_text>Random Forest<jupyter_code>df_plot = df_data_3[df_data_3['Model'].isin(['RF'])] 
fig, ax = plt.subplots(figsize=a4_dims)
ax = sns.lineplot(x="Thresholds", y="Score", hue="Resolution", style="Score Type", palette=palette_3, 
                 legend="full", data=df_plot)
#plt.legend(loc='center left', bbox_to_anchor=(1.0, 0.5))
plt.savefig('./results/graphics/ST_RF.pdf', format='pdf', bbox_inches='tight')
plt.show()<jupyter_output><empty_output><jupyter_text>MultiLayer Perceptron<jupyter_code>df_plot = df_data_3[df_data_3['Model'].isin(['MLP'])]
fig, ax = plt.subplots(figsize=a4_dims)
ax = sns.lineplot(x="Thresholds", y="Score", hue="Resolution", style="Score Type", palette=palette_3, 
                  legend="full", data= df_plot)
#plt.legend(loc='center left', bbox_to_anchor=(1.0, 0.5))
plt.savefig('./results/graphics/ST_MLP.pdf', format='pdf', bbox_inches='tight')
plt.show()<jupyter_output><empty_output><jupyter_text>Precision Vs Recall All Models and all Resolutions<jupyter_code>df_plot = df_data_2
fig, ax = plt.subplots(figsize=a4_dims)
ax = sns.lineplot(x="Recalls", y="Precisions", hue="Model", style="Resolution",
                  palette=palette_4, legend="full", data=df_plot)
#plt.legend(loc='center left', bbox_to_anchor=(1.0, 0.45))
ax.set_xlim(0.7,1.01)
plt.savefig('./results/graphics/PR.pdf', format='pdf', bbox_inches='tight')
plt.show()<jupyter_output><empty_output><jupyter_text>Logistic Regression<jupyter_code>df_plot = df_data_2[df_data_2['Model']=='LR']
fig, ax = plt.subplots(figsize=a4_dims)
ax = sns.lineplot(x="Recalls", y="Precisions", hue="Resolution",
                  palette=palette_3, legend="full", data=df_plot)
#plt.legend(loc='center left', bbox_to_anchor=(1.0, 0.45))
ax.set_xlim(0.4,1.05)
plt.savefig('./results/graphics/PR_LR.pdf', format='pdf', bbox_inches='tight')
plt.show()<jupyter_output><empty_output><jupyter_text>Support Vector Machine<jupyter_code>df_plot = df_data_2[df_data_2['Model']=='SVC']
fig, ax = plt.subplots(figsize=a4_dims)
ax = sns.lineplot(x="Recalls", y="Precisions", hue="Resolution", palette=palette_3, 
                  legend="full", data=df_plot)
#plt.legend(loc='center left', bbox_to_anchor=(1.0, 0.45))
ax.set_xlim(0.2,1.05)
plt.savefig('./results/graphics/PR_SVC.pdf', format='pdf', bbox_inches='tight')
plt.show()<jupyter_output><empty_output><jupyter_text>Random Forest<jupyter_code>df_plot = df_data_2[df_data_2['Model']=='RF']
fig, ax = plt.subplots(figsize=a4_dims)
ax = sns.lineplot(x="Recalls", y="Precisions", hue="Resolution", palette=palette_3, 
                  legend="full", data=df_plot)
#plt.legend(loc='center left', bbox_to_anchor=(1.0, 0.45))
ax.set_xlim(0.4,1.05)
plt.savefig('./results/graphics/PR_RF.pdf', format='pdf', bbox_inches='tight')
plt.show()<jupyter_output><empty_output><jupyter_text>MultiLayer Perceptron<jupyter_code>df_plot = df_data_2[df_data_2['Model']=='MLP']
fig, ax = plt.subplots(figsize=a4_dims)
ax = sns.lineplot(x="Recalls", y="Precisions", hue="Resolution", palette=palette_3, 
                  legend="full", data=df_plot)
#plt.legend(loc='center left', bbox_to_anchor=(1.0, 0.45))
ax.set_xlim(0.7,1.02)
plt.savefig('./results/graphics/PR_MLP.pdf', format='pdf', bbox_inches='tight')
plt.show()<jupyter_output><empty_output><jupyter_text>ROC Curve All Models and all Resolutions<jupyter_code>palette = sns.color_palette(flatui)
df_plot = df_data_2
fig, ax = plt.subplots(figsize=a4_dims)
ax = sns.lineplot(x="False Positives Rate", y="True Positive Rate", hue="Model", palette=palette_4,
                 style="Resolution", legend="full", data=df_plot)
#plt.legend(loc='center left', bbox_to_anchor=(1.0, 0.45))
ax.set_xlim(-0.02,0.3)
ax.set_ylim(0.5,1.02)
plt.plot([0,1],[0,1], 'k--')
plt.savefig('./results/graphics/ROC.pdf', format='pdf', bbox_inches='tight')
plt.show()<jupyter_output><empty_output><jupyter_text>Logistic Regression<jupyter_code>palette = sns.color_palette("mako_r",3)
df_plot = df_data_2[df_data_2['Model']=='LR']
fig, ax = plt.subplots(figsize=a4_dims)
ax = sns.lineplot(x="False Positives Rate", y="True Positive Rate", hue="Resolution", palette=palette_3, 
                  legend="full", data=df_plot)
#plt.legend(loc='center left', bbox_to_anchor=(1.0, 0.45))
#ax.set_xlim(0.7,1.02)
plt.plot([0,1],[0,1], 'k--')
plt.savefig('./results/graphics/ROC_LR.pdf', format='pdf', bbox_inches='tight')
plt.show()<jupyter_output><empty_output><jupyter_text>Support Vector Machine<jupyter_code>df_plot = df_data_2[df_data_2['Model']=='SVC']
fig, ax = plt.subplots(figsize=a4_dims)
ax = sns.lineplot(x="False Positives Rate", y="True Positive Rate", hue="Resolution", palette=palette_3, 
                  legend="full", data=df_plot)
#plt.legend(loc='center left', bbox_to_anchor=(1.0, 0.45))
#ax.set_xlim(0.7,1.02)
plt.plot([0,1],[0,1], 'k--')
plt.savefig('./results/graphics/ROC_SVC.pdf', format='pdf', bbox_inches='tight')
plt.show()<jupyter_output><empty_output><jupyter_text>Random Forest<jupyter_code>df_plot = df_data_2[df_data_2['Model']=='RF']
fig, ax = plt.subplots(figsize=a4_dims)
ax = sns.lineplot(x="False Positives Rate", y="True Positive Rate", hue="Resolution", palette=palette_3, 
                  legend="full", data=df_plot)
#plt.legend(loc='center left', bbox_to_anchor=(1.0, 0.45))
#ax.set_xlim(0.7,1.02)
plt.plot([0,1],[0,1], 'k--')
plt.savefig('./results/graphics/ROC_RF.pdf', format='pdf', bbox_inches='tight')
plt.show()<jupyter_output><empty_output><jupyter_text>MultiLayer Perceptron<jupyter_code>df_plot = df_data_2[df_data_2['Model']=='MLP']
fig, ax = plt.subplots(figsize=a4_dims)
ax = sns.lineplot(x="False Positives Rate", y="True Positive Rate", hue="Resolution", palette=palette_3, 
                  legend="full", data=df_plot)
#plt.legend(loc='center left', bbox_to_anchor=(1.0, 0.45))
#ax.set_xlim(0.7,1.02)
plt.plot([0,1],[0,1], 'k--')
plt.savefig('./results/graphics/ROC_MLP.pdf', format='pdf', bbox_inches='tight')
plt.show()<jupyter_output><empty_output><jupyter_text>Precision and Recall vs Thresholds All Models Low Resolution<jupyter_code>palette = sns.color_palette("mako_r",4)
df_plot = df_data_3[df_data_3['Resolution']=='Low']
fig, ax = plt.subplots(figsize=a4_dims)
ax = sns.lineplot(x="Thresholds", y="Score", hue="Model", style="Score Type", palette=palette_4, 
                  data=df_plot)
#plt.legend(loc='center left', bbox_to_anchor=(1.0, 0.5))
ax.set_xlim(-1,4)
plt.savefig('./results/graphics/ST_LOW.pdf', format='pdf', bbox_inches='tight')
plt.show()<jupyter_output><empty_output><jupyter_text>All Models Medium Resolution<jupyter_code>df_plot = df_data_3[df_data_3['Resolution']=='Med']
fig, ax = plt.subplots(figsize=a4_dims)
ax = sns.lineplot(x="Thresholds", y="Score", hue="Model", style="Score Type", palette=palette_4, 
                  data=df_plot)
#plt.legend(loc='center left', bbox_to_anchor=(1.0, 0.5))
ax.set_xlim(-1,4)
plt.savefig('./results/graphics/ST_MED.pdf', format='pdf', bbox_inches='tight')
plt.show()<jupyter_output><empty_output><jupyter_text>All Models High Resolution<jupyter_code>df_plot = df_data_3[df_data_3['Resolution']=='High']
fig, ax = plt.subplots(figsize=a4_dims)
ax = sns.lineplot(x="Thresholds", y="Score", hue="Model", style="Score Type", palette=palette_4, 
                  data=df_plot)
#plt.legend(loc='center left', bbox_to_anchor=(1.0, 0.5))
ax.set_xlim(-1,4)
plt.savefig('./results/graphics/ST_HIGH.pdf', format='pdf', bbox_inches='tight')
plt.show()<jupyter_output><empty_output><jupyter_text>Precision VS Recalls All Models Low Resolution<jupyter_code>df_plot = df_data_2[df_data_2['Resolution']=='Low']
fig, ax = plt.subplots(figsize=a4_dims)
ax = sns.lineplot(x="Recalls", y="Precisions", hue="Model", palette=palette_4,
                  legend="full", sizes=(1,4), data=df_plot)
#plt.legend(loc='center left', bbox_to_anchor=(1.0, 0.45))
ax.set_xlim(0.4,1.02)
plt.savefig('./results/graphics/PR_LOW.pdf', format='pdf', bbox_inches='tight')
plt.show()<jupyter_output><empty_output><jupyter_text>All Models Medium Resolution<jupyter_code>df_plot = df_data_2[df_data_2['Resolution']=='Med']
fig, ax = plt.subplots(figsize=a4_dims)
ax = sns.lineplot(x="Recalls", y="Precisions", hue="Model", palette=palette_4, 
                  legend="full", sizes=(1,4), data=df_plot)
#plt.legend(loc='center left', bbox_to_anchor=(1.0, 0.45))
ax.set_xlim(0.3,1.02)
plt.savefig('./results/graphics/PR_MED.pdf', format='pdf', bbox_inches='tight')
plt.show()<jupyter_output><empty_output><jupyter_text>All Models High Resolution<jupyter_code>df_plot = df_data_2[df_data_2['Resolution']=='High']
fig, ax = plt.subplots(figsize=a4_dims)
ax = sns.lineplot(x="Recalls", y="Precisions", hue="Model", palette=palette_4, 
                  legend="full", sizes=(1,4), data=df_plot)
#plt.legend(loc='center left', bbox_to_anchor=(1.0, 0.45))
ax.set_xlim(0.4,1.02)
plt.savefig('./results/graphics/PR_HIGH.pdf', format='pdf', bbox_inches='tight')
plt.show()<jupyter_output><empty_output><jupyter_text>ROC Curve and Area Under Curve All Models Low Resolution<jupyter_code>df_plot = df_data_2[df_data_2['Resolution']=='Low']
df_plot['ROC AUC'] = np.round_(df_plot['ROC AUC'],decimals=3)
fig, ax = plt.subplots(figsize=a4_dims)
ax = sns.lineplot(x="False Positives Rate", y="True Positive Rate", hue="Model", palette=palette_4, 
                  size="ROC AUC", legend="full", data=df_plot)
#plt.legend(loc='center left', bbox_to_anchor=(1.0, 0.45))
#ax.set_xlim(0.7,1.02)
plt.plot([0,1],[0,1], 'k--')
plt.savefig('./results/graphics/ROC_LOW.pdf', format='pdf', bbox_inches='tight')
plt.show()<jupyter_output>/home/juanfe/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy<jupyter_text>All Models Medium Resolution<jupyter_code>df_plot = df_data_2[df_data_2['Resolution']=='Med']
df_plot['ROC AUC'] = np.round_(df_plot['ROC AUC'],decimals=3)
fig, ax = plt.subplots(figsize=a4_dims)
ax = sns.lineplot(x="False Positives Rate", y="True Positive Rate", hue="Model", palette=palette_4, 
                  size="ROC AUC", legend="full", data=df_plot)
#plt.legend(loc='center left', bbox_to_anchor=(1.0, 0.45))
#ax.set_xlim(0.7,1.02)
plt.plot([0,1],[0,1], 'k--')
plt.savefig('./results/graphics/ROC_MED.pdf', format='pdf', bbox_inches='tight')
plt.show()<jupyter_output>/home/juanfe/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy<jupyter_text>All Models High Resolution<jupyter_code>df_plot = df_data_2[df_data_2['Resolution']=='High']
df_plot['ROC AUC'] = np.round_(df_plot['ROC AUC'],decimals=3)
fig, ax = plt.subplots(figsize=a4_dims)
ax = sns.lineplot(x="False Positives Rate", y="True Positive Rate", hue="Model", palette=palette_4, 
                  size="ROC AUC", legend="full", data=df_plot)
#plt.legend(loc='center left', bbox_to_anchor=(1.0, 0.45))
#ax.set_xlim(0.7,1.02)
plt.plot([0,1],[0,1], 'k--')
plt.savefig('./results/graphics/ROC_HIGH.pdf', format='pdf', bbox_inches='tight')
plt.show()<jupyter_output>/home/juanfe/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
{'fraction_non_alphanumeric': 0.17362716391842606, 'fraction_numerical': 0.02167771099809935, 'mean_word_length': 5.962804005722461, 'pattern_counts': {'":': 0, '<': 107, '<?xml version=': 0, '>': 107, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '27', 'dir': 'jupyter-structured-clean-dedup', 'id': '581242', 'license': 'MIT', 'n_tokens_mistral': 8070, 'n_tokens_neox': 7626, 'n_words': 1129, 'path': '4_data_analysis.ipynb', 'repo_name': 'jflatorreg/MAAD-Workflow', 'size': '19499'}