<jupyter_start><jupyter_text>ANI-1x trained to different datasets tested for extensibility<jupyter_code>tb = eed.get_totalerror_table() # All errors combined
meanMAE = np.mean(np.stack([tb['$\Delta$E$_\mathrm{MAE}$$\mu$'].values,
                            tb['E$_\mathrm{MAE}$$\mu$'].values,
                            tb['F$_\mathrm{MAE}$$\mu$'].values]),axis=0)

tb['MeanMAE'] = meanMAE

meanRMS = np.mean(np.stack([tb['$\Delta$E$_\mathrm{RMS}$$\mu$'].values,
                            tb['E$_\mathrm{RMS}$$\mu$'].values,
                            tb['F$_\mathrm{RMS}$$\mu$'].values]),axis=0)

tb['MeanRMS'] = meanRMS

tb['TotalMean'] = np.mean([meanMAE,meanRMS],axis=0)
# Set CSS properties for th elements in dataframe
th_props = [
  ('font-size', '11px'),
  ('text-align', 'center'),
  ('font-weight', 'bold'),
  ('color', '#6d6d6d'),
  ('background-color', '#f7f7f9')
  ]

# Set CSS properties for td elements in dataframe
td_props = [
  ('font-size', '11px')
  ]

# Set table styles
styles = [
  dict(selector="th", props=th_props),
  dict(selector="td", props=td_props)
  ]

# Set colormap equal to seaborns light green color palette
cm = ListedColormap(sns.color_palette("Greens"))

(tb.style
  .background_gradient(cmap=cm, subset=['$\Delta$E$_\mathrm{MAE}$$\mu$',
                                        '$\Delta$E$_\mathrm{RMS}$$\mu$',
                                        'E$_\mathrm{MAE}$$\mu$',
                                        'E$_\mathrm{RMS}$$\mu$',
                                        'F$_\mathrm{MAE}$$\mu$',
                                        'F$_\mathrm{RMS}$$\mu$',
                                        'MeanRMS',
                                        'MeanMAE',
                                        'TotalMean'])
  .highlight_min(subset=['$\Delta$E$_\mathrm{MAE}$$\mu$',
                         '$\Delta$E$_\mathrm{RMS}$$\mu$',
                         'E$_\mathrm{MAE}$$\mu$',
                         'E$_\mathrm{RMS}$$\mu$',
                         'F$_\mathrm{MAE}$$\mu$',
                         'F$_\mathrm{RMS}$$\mu$',
                         'MeanRMS',
                         'MeanMAE',
                         'TotalMean'])
  .set_caption('COMP6')
  .format({'total_amt_usd_pct_diff': "{:.2%}"})
  .set_table_styles(styles))
tb = eed.get_error_table(tskey='DrugBank')
meanMAE = np.mean(np.stack([tb['$\Delta$E$_\mathrm{MAE}$$\mu$'].values,
                            tb['E$_\mathrm{MAE}$$\mu$'].values,
                            tb['F$_\mathrm{MAE}$$\mu$'].values]),axis=0)

tb['MeanMAE'] = meanMAE

meanRMS = np.mean(np.stack([tb['$\Delta$E$_\mathrm{RMS}$$\mu$'].values,
                            tb['E$_\mathrm{RMS}$$\mu$'].values,
                            tb['F$_\mathrm{RMS}$$\mu$'].values]),axis=0)

tb['MeanRMS'] = meanRMS

tb['TotalMean'] = np.mean([meanMAE,meanRMS],axis=0)
# Set CSS properties for th elements in dataframe
th_props = [
  ('font-size', '11px'),
  ('text-align', 'center'),
  ('font-weight', 'bold'),
  ('color', '#6d6d6d'),
  ('background-color', '#f7f7f9')
  ]

# Set CSS properties for td elements in dataframe
td_props = [
  ('font-size', '11px')
  ]

# Set table styles
styles = [
  dict(selector="th", props=th_props),
  dict(selector="td", props=td_props)
  ]

# Set colormap equal to seaborns light green color palette
cm = ListedColormap(sns.color_palette("Greens"))

(tb.style
  .background_gradient(cmap=cm, subset=['$\Delta$E$_\mathrm{MAE}$$\mu$',
                                        '$\Delta$E$_\mathrm{RMS}$$\mu$',
                                        'E$_\mathrm{MAE}$$\mu$',
                                        'E$_\mathrm{RMS}$$\mu$',
                                        'F$_\mathrm{MAE}$$\mu$',
                                        'F$_\mathrm{RMS}$$\mu$',
                                        'MeanRMS',
                                        'MeanMAE',
                                        'TotalMean'])
  .highlight_min(subset=['$\Delta$E$_\mathrm{MAE}$$\mu$',
                         '$\Delta$E$_\mathrm{RMS}$$\mu$',
                         'E$_\mathrm{MAE}$$\mu$',
                         'E$_\mathrm{RMS}$$\mu$',
                         'F$_\mathrm{MAE}$$\mu$',
                         'F$_\mathrm{RMS}$$\mu$',
                         'MeanRMS',
                         'MeanMAE',
                         'TotalMean'])
  .set_caption('Drugbank')
  .format({'total_amt_usd_pct_diff': "{:.2%}"})
  .set_table_styles(styles))
tb = eed.get_error_table(tskey='Tripeptide')
meanMAE = np.mean(np.stack([tb['$\Delta$E$_\mathrm{MAE}$$\mu$'].values,
                            tb['E$_\mathrm{MAE}$$\mu$'].values,
                            tb['F$_\mathrm{MAE}$$\mu$'].values]),axis=0)

tb['MeanMAE'] = meanMAE

meanRMS = np.mean(np.stack([tb['$\Delta$E$_\mathrm{RMS}$$\mu$'].values,
                            tb['E$_\mathrm{RMS}$$\mu$'].values,
                            tb['F$_\mathrm{RMS}$$\mu$'].values]),axis=0)

tb['MeanRMS'] = meanRMS

tb['TotalMean'] = np.mean([meanMAE,meanRMS],axis=0)
# Set CSS properties for th elements in dataframe
th_props = [
  ('font-size', '11px'),
  ('text-align', 'center'),
  ('font-weight', 'bold'),
  ('color', '#6d6d6d'),
  ('background-color', '#f7f7f9')
  ]

# Set CSS properties for td elements in dataframe
td_props = [
  ('font-size', '11px')
  ]

# Set table styles
styles = [
  dict(selector="th", props=th_props),
  dict(selector="td", props=td_props)
  ]

# Set colormap equal to seaborns light green color palette
cm = ListedColormap(sns.color_palette("Greens"))

(tb.style
  .background_gradient(cmap=cm, subset=['$\Delta$E$_\mathrm{MAE}$$\mu$',
                                        '$\Delta$E$_\mathrm{RMS}$$\mu$',
                                        'E$_\mathrm{MAE}$$\mu$',
                                        'E$_\mathrm{RMS}$$\mu$',
                                        'F$_\mathrm{MAE}$$\mu$',
                                        'F$_\mathrm{RMS}$$\mu$',
                                        'MeanRMS',
                                        'MeanMAE',
                                        'TotalMean'])
  .highlight_min(subset=['$\Delta$E$_\mathrm{MAE}$$\mu$',
                         '$\Delta$E$_\mathrm{RMS}$$\mu$',
                         'E$_\mathrm{MAE}$$\mu$',
                         'E$_\mathrm{RMS}$$\mu$',
                         'F$_\mathrm{MAE}$$\mu$',
                         'F$_\mathrm{RMS}$$\mu$',
                         'MeanRMS',
                         'MeanMAE',
                         'TotalMean'])
  .set_caption('Tripeptides')
  .format({'total_amt_usd_pct_diff': "{:.2%}"})
  .set_table_styles(styles))
tb = eed.get_error_table(tskey='MDBenchmark')
meanMAE = np.mean(np.stack([tb['$\Delta$E$_\mathrm{MAE}$$\mu$'].values,
                            tb['E$_\mathrm{MAE}$$\mu$'].values,
                            tb['F$_\mathrm{MAE}$$\mu$'].values]),axis=0)

tb['MeanMAE'] = meanMAE

meanRMS = np.mean(np.stack([tb['$\Delta$E$_\mathrm{RMS}$$\mu$'].values,
                            tb['E$_\mathrm{RMS}$$\mu$'].values,
                            tb['F$_\mathrm{RMS}$$\mu$'].values]),axis=0)

tb['MeanRMS'] = meanRMS

tb['TotalMean'] = np.mean([meanMAE,meanRMS],axis=0)
# Set CSS properties for th elements in dataframe
th_props = [
  ('font-size', '11px'),
  ('text-align', 'center'),
  ('font-weight', 'bold'),
  ('color', '#6d6d6d'),
  ('background-color', '#f7f7f9')
  ]

# Set CSS properties for td elements in dataframe
td_props = [
  ('font-size', '11px')
  ]

# Set table styles
styles = [
  dict(selector="th", props=th_props),
  dict(selector="td", props=td_props)
  ]

# Set colormap equal to seaborns light green color palette
cm = ListedColormap(sns.color_palette("Greens"))

(tb.style
  .background_gradient(cmap=cm, subset=['$\Delta$E$_\mathrm{MAE}$$\mu$',
                                        '$\Delta$E$_\mathrm{RMS}$$\mu$',
                                        'E$_\mathrm{MAE}$$\mu$',
                                        'E$_\mathrm{RMS}$$\mu$',
                                        'F$_\mathrm{MAE}$$\mu$',
                                        'F$_\mathrm{RMS}$$\mu$',
                                        'MeanRMS',
                                        'MeanMAE',
                                        'TotalMean'])
  .highlight_min(subset=['$\Delta$E$_\mathrm{MAE}$$\mu$',
                         '$\Delta$E$_\mathrm{RMS}$$\mu$',
                         'E$_\mathrm{MAE}$$\mu$',
                         'E$_\mathrm{RMS}$$\mu$',
                         'F$_\mathrm{MAE}$$\mu$',
                         'F$_\mathrm{RMS}$$\mu$',
                         'MeanRMS',
                         'MeanMAE',
                         'TotalMean'])
  .set_caption('MDBenchMark')
  .format({'total_amt_usd_pct_diff': "{:.2%}"})
  .set_table_styles(styles))
tb = eed.get_error_table(tskey='s66x8')
meanMAE = np.mean(np.stack([tb['$\Delta$E$_\mathrm{MAE}$$\mu$'].values,
                            tb['E$_\mathrm{MAE}$$\mu$'].values,
                            tb['F$_\mathrm{MAE}$$\mu$'].values]),axis=0)

tb['MeanMAE'] = meanMAE

meanRMS = np.mean(np.stack([tb['$\Delta$E$_\mathrm{RMS}$$\mu$'].values,
                            tb['E$_\mathrm{RMS}$$\mu$'].values,
                            tb['F$_\mathrm{RMS}$$\mu$'].values]),axis=0)

tb['MeanRMS'] = meanRMS

tb['TotalMean'] = np.mean([meanMAE,meanRMS],axis=0)
# Set CSS properties for th elements in dataframe
th_props = [
  ('font-size', '11px'),
  ('text-align', 'center'),
  ('font-weight', 'bold'),
  ('color', '#6d6d6d'),
  ('background-color', '#f7f7f9')
  ]

# Set CSS properties for td elements in dataframe
td_props = [
  ('font-size', '11px')
  ]

# Set table styles
styles = [
  dict(selector="th", props=th_props),
  dict(selector="td", props=td_props)
  ]

# Set colormap equal to seaborns light green color palette
cm = ListedColormap(sns.color_palette("Greens"))

(tb.style
  .background_gradient(cmap=cm, subset=['$\Delta$E$_\mathrm{MAE}$$\mu$',
                                        '$\Delta$E$_\mathrm{RMS}$$\mu$',
                                        'E$_\mathrm{MAE}$$\mu$',
                                        'E$_\mathrm{RMS}$$\mu$',
                                        'F$_\mathrm{MAE}$$\mu$',
                                        'F$_\mathrm{RMS}$$\mu$',
                                        'MeanRMS',
                                        'MeanMAE',
                                        'TotalMean'])
  .highlight_min(subset=['$\Delta$E$_\mathrm{MAE}$$\mu$',
                         '$\Delta$E$_\mathrm{RMS}$$\mu$',
                         'E$_\mathrm{MAE}$$\mu$',
                         'E$_\mathrm{RMS}$$\mu$',
                         'F$_\mathrm{MAE}$$\mu$',
                         'F$_\mathrm{RMS}$$\mu$',
                         'MeanRMS',
                         'MeanMAE',
                         'TotalMean'])
  .set_caption('MDBenchMark')
  .format({'total_amt_usd_pct_diff': "{:.2%}"})
  .set_table_styles(styles))
eed.get_error_table(tskey='GDB-10to13')
eed.get_error_table(tskey='MDBenchmark')
eed.get_error_table(tskey='s66x8')
#eed.get_error_table(tskey='s66x8')
# Create a Pandas Excel writer using XlsxWriter as the engine.
writer = pd.ExcelWriter(xls_store, engine='xlsxwriter')

# Build latex document
header = r'\documentclass{article}'+'\n'+r'\usepackage{booktabs}'+'\n'+r'\usepackage[margin=1.0in]{geometry}'+'\n'+r'\begin{document}'+'\n'

begtab = r'\begin{table}'+'\n'+r'\centering'+'\n'+r'\tiny'+'\n'
endtab = r'\end{table}'+'\n'
enddoc = r'\end{document}'
styles = []
with open(tex_store, "w") as text_file:
    text_file.write(header)
    print('Stats for: Complete benchmark')
    etab = eed.get_totalerror_table()
    styles.append(etab.style)
    text_file.write(begtab+r'\caption{Data set: Complete benchmark}'+'\n'+etab.to_latex(escape=False)+endtab)

    # Convert the dataframe to an XlsxWriter Excel object.
    etab.to_excel(writer, sheet_name="Complete Benchmark")

    paetab = eed.get_totalerrorperatom_table()
    text_file.write(begtab+r'\caption{Data set: Complete benchmark per atom}'+'\n'+paetab.to_latex(escape=False)+endtab)
    paetab.to_excel(writer, sheet_name="Complete Benchmark per atom")
    
    
    for k in tsname:
        print('Stats for:',k)
        etab = eed.get_error_table(k)
        etab.to_excel(writer, sheet_name=k)
        styles.append(etab.style)
        text_file.write(begtab+r'\caption{Data set: '+k+r'}'+'\n'+etab.to_latex(escape=False)+endtab)
    text_file.write(enddoc)
styles[0]

# Close the Pandas Excel writer and output the Excel file.
writer.save()
cmap = mpl.cm.nipy_spectral
for i in range(1,16):
    print('Models:',str(i+1).zfill(2))
    eed.determine_min_error_by_sigma('ANI-1xTL'+str(i+1).zfill(2),
                                     0.5, 50.0,
                                     labelx=r'$\mathrm{\rho}$ [$\mathrm{kcal \times mol^{-1} \times (\sqrt{N_a})^{-1}}}$]',
                                     labely=r'$\mathrm{\mathcal{E}}$ [$\mathrm{kcal \times mol^{-1} \times (\sqrt{N_a})^{-1}}$]',
                                     xyrange=(0.0,2.0,0.0,12.0),
                                     figsize=(12.0, 9.0),
                                     cmap=cmap,
                                     tskeys=['GDB-10to13'],
                                     #storepath='/home/jsmith48/scratch/error_stats/rho_selection_6.0.0.pdf',
                                     )
cmap = mpl.cm.nipy_spectral
eed.determine_min_error_by_sigma('ANI-1xTL_lr-5',
                                 0.5, 50.0,
                                 labelx=r'$\mathrm{\rho}$ [$\mathrm{kcal \times mol^{-1} \times (\sqrt{N_a})^{-1}}}$]',
                                 labely=r'$\mathrm{\mathcal{E}}$ [$\mathrm{kcal \times mol^{-1} \times (\sqrt{N_a})^{-1}}$]',
                                 xyrange=(0.0,2.0,0.0,12.0),
                                 figsize=(12.0, 9.0),
                                 cmap=cmap,
                                 tskeys=['GDB-10to13'],
                                 #storepath='/home/jsmith48/scratch/error_stats/rho_selection_6.0.0.pdf',
                                 )
print(eed.get_net_keys())
#print(eed.get_size("9.0.5","MDBenchmark"))
#print(eed.get_size("9.0.4","Tripeptide"))
#print(eed.get_size("9.0.4","GDB07to09"))
#print(eed.get_size("9.0.4","GDB10to13"))
# cmap = mpl.cm.nipy_spectral
# Erms  = []
# dErms = []
# for i in range(1,17):
#     print('Models:',str(i).zfill(2))
#     Erms.append(eed.generate_rmserror('ANI-1xTL'+str(i).zfill(2),'GDB-10to13','Eani','Edft'))
#     dErms.append(eed.generate_rmserror('ANI-1xTL'+str(i).zfill(2),'GDB-10to13','dEani','dEdft'))

eed.generate_correlation_plot('ANI-2x_3','DrugBk (CHNO)','Fani','Fdft')
eed.generate_correlation_plot('ANI-2x_3','DrugBk (SFCl)','Fani','Fdft')
#eed.generate_correlation_plot('ANI-2x_3-100','DrugBk (CHNO)','dEani','dEdft')

#eed.generate_correlation_plot('6.0.0','MDBenchmark','Fani','Fdft')
#cmap = mpl.cm.nipy_spectral
#eed.generate_correlation_plot('ANI-0707.0001.0410_bt5','GDB-11 (SFCl)','Fani','Fdft',cmap=cmap)
#eed.generate_correlation_plot('0707.0001.0402','GDB-07 (SFCl)','dEani','dEdft')
#eed.generate_correlation_plot('0707.0001.0402','GDB-08 (SFCl)','dEani','dEdft')
#eed.generate_correlation_plot('0707.0001.0402','GDB-09 (SFCl)','dEani','dEdft')
#eed.generate_correlation_plot('0707.0001.0402','GDB-10 (SFCl)','dEani','dEdft')
#eed.generate_correlation_plot('0707.0001.0402','GDB-11 (SFCl)','dEani','dEdft')
#eed.generate_correlation_plot('AL6','DrugBank','Fani','Fdft')
#eed.generate_correlation_plot('ARC1','MDBenchmark','Fani','Fdft')
#eed.generate_correlation_plot('ARC8','MDBenchmark','Fani','Fdft')
import numpy as np
from matplotlib import gridspec 

fig = plt.subplots(figsize=[15,6])
gs  = gridspec.GridSpec(1, 2, width_ratios=[0.5, 0.5])

ax1 = plt.subplot(gs[0])
ax2 = plt.subplot(gs[1])

ax1.plot(np.arange(1,17),Erms,marker='o')
ax1.set_title(r'E improvment with ensemble size')
ax1.set_xlabel('Number of members')
ax1.set_ylabel('RMSE error (kcal/mol)')

ax2.plot(np.arange(1,17),dErms,marker='o')
ax2.set_title(r'$\Delta$E improvment with ensemble size')
ax2.set_xlabel('Number of members')
ax2.set_ylabel('RMSE error (kcal/mol)')

plt.show()
#eed.plot_bar_propsbynet([('Total Energy',r'RMSE ($kcal \times mol^{-1}$)','Eani','Edft',[0.0,23.0]),
#                         ('Relative Energy',r'RMSE ($kcal \times mol^{-1}$)','dEani', 'dEdft',[0.0,12.0]),
#                         ('Force',r'RMSE ($kcal \times mol^{-1} \times \AA^{-1}$)','Fani', 'Fdft',[0.0,10.0])], 
#                         ['GDB07to09', 'GDB10to13', 'DrugBank', 'Tripeptide', 's66x8'],
#                         ntwks=['6.0.0','7.0.4','8.0.6','8.3.6','9.0.4','ANI-1'],
#                         fontsize=30, bbox_to_anchor=(0.3, 1.0), figsize=(40.0, 32.0), ncol=2, errortype='RMSE')
eed.plot_error_by_net([(r'$E (kcal \times mol^{-1})$',r'RMSE ($kcal \times mol^{-1}$)','Eani','Edft'),
                       (r'$\Delta E (kcal \times mol^{-1})$',r'RMSE ($kcal \times mol^{-1}$)','dEani','dEdft'),
                       (r'$F (kcal \times mol^{-1}\AA^{-1})$',r'RMSE ($kcal \times mol^{-1}$)','Fani','Fdft'),], 
                         ['DHL_DAT.0001', 'DHL_DAT.0002','DHL_DAT.0003',],
                         ntwks=['ANI-1x-DHL-0000.0000','ANI-1x-DHL-0000.0001','ANI-1x-DHL-0000.0002'],
                         fontsize=20, bbox_to_anchor=(1.0, 1.0), figsize=(30.0, 15.0), ncol=1, errortype='RMSE',
                        #storepath='/home/jujuman/Dropbox/ANAKIN-ME-RESEARCH/PUBLICATION-ActiveLearning/Figures/figure_TSALvsANI_RMSE.pdf',
                      )
eed.plot_error_by_net([(r'$E (kcal \times mol^{-1})$',r'RMSE ($kcal \times mol^{-1}$)','Eani','Edft'),
                       (r'$\Delta E (kcal \times mol^{-1})$',r'RMSE ($kcal \times mol^{-1}$)','dEani','dEdft'),
                       (r'$F (kcal \times mol^{-1}\AA^{-1})$',r'RMSE ($kcal \times mol^{-1}$)','Fani','Fdft'),], 
                         ['GDB07to09', 'GDB10to13', 'DrugBank', 'Tripeptide', 'MDBenchmark', 's66x8'],
                         ntwks=['AL1', 'AL2', 'AL3', 'AL4', 'AL5', 'AL6', 'ANI-1'],
                         fontsize=20, bbox_to_anchor=(1.0, 1.0), figsize=(30.0, 15.0), ncol=1, errortype='RMSE',
                        #storepath='/home/jujuman/Dropbox/ANAKIN-ME-RESEARCH/PUBLICATION-ActiveLearning/Figures/figure_TSALvsANI_RMSE.pdf')
eed.plot_error_by_net([(r'$E (kcal \times mol^{-1})$',r'RMSE ($kcal \times mol^{-1}$)','Eani','Edft'),
                       (r'$\Delta E (kcal \times mol^{-1})$',r'RMSE ($kcal \times mol^{-1}$)','dEani','dEdft'),
                       (r'$F (kcal \times mol^{-1}\AA^{-1})$',r'RMSE ($kcal \times mol^{-1}$)','Fani','Fdft'),], 
                         ['GDB07to09', 'GDB10to13', 'DrugBank', 'Tripeptide', 'MDBenchmark', 's66x8'],
                         ntwks=['AL1', 'AL2', 'AL3', 'AL4', 'AL5', 'AL6', 'ANI-1'],
                         fontsize=20, bbox_to_anchor=(1.0, 1.0), figsize=(30.0, 15.0), ncol=1, errortype='MAE',
                        #storepath='/home/jujuman/Dropbox/ANAKIN-ME-RESEARCH/PUBLICATION-ActiveLearning/Figures/figure_TSALvsANI_MAE.pdf')
#eed.generate_violin_distribution('GDB10to13')
'''
eed.plot_error_by_net([(r'$E_T (kcal \times mol^{-1})$',r'RMSE ($kcal \times mol^{-1}$)','Eani','Edft'),
                       (r'$\Delta E (kcal \times mol^{-1})$',r'RMSE ($kcal \times mol^{-1}$)','dEani','dEdft'),
                       (r'$F (kcal \times mol^{-1}\AA^{-1})$',r'RMSE ($kcal \times mol^{-1}$)','Fani','Fdft'),], 
                         ['GDB07to09', 'GDB10to13', 'DrugBank', 'Tripeptide', 'MDBenchmark', 's66x8'],
                         ntwks=['ARC1','ARC2','ARC3','ARC4','ARC5','ARC6','ARC7','ARC8'],
                        fontsize=20, bbox_to_anchor=(1.0, 1.0), figsize=(20.0, 30.0), ncol=1, errortype='RMSE',
                        storepath='/home/jujuman/Research/DataReductionMethods/train_test/tables/figure_TSALvsANI_RMSE.pdf')
'''
Edft1 = eed.get_range_stats(tslist=['DHL_DAT.0002'],dkey='Fdft')
plt.hist(Edft1,bins=20, log=True, edgecolor='black')
plt.show()
Edft1 = eed.get_range_stats(tslist=['GDB-10to13 (CHNO)'],dkey='Na')
plt.hist(Edft1,bins=18, log=True, edgecolor='black')
plt.show()
Edft2 = eed.get_range_stats(tslist=['r10_ccsd'],dkey='Edft')
plt.hist(Edft2,bins=16,color='r', log=True, edgecolor='black')
plt.show()
Edft2 = eed.get_range_stats(tslist=['r10_ccsd'],dkey='Na')
plt.hist(Edft2,bins=16, log=True,color='r', edgecolor='black')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.204656803298569, 'fraction_numerical': 0.028619936939121998, 'mean_word_length': 2.1085645355850424, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 58, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '285918', 'license': 'MIT', 'n_tokens_mistral': 7673, 'n_tokens_neox': 7113, 'n_words': 900, 'path': 'notebooks/evaluate_tsdata.ipynb', 'repo_name': 'vwcruzeiro/ANI-Tools', 'size': '20615'}