<jupyter_start><jupyter_text>Management - Gen 2 Opt<jupyter_code>import qcportal as ptl
import pandas as pd
import datetime
import time
import numpy as np

from management import *<jupyter_output><empty_output><jupyter_text>connect without auth read onlyclient = ptl.FractalClient()<jupyter_code># connect with authentication, therefore write access
# don't use unless you plan to submit things
client = ptl.FractalClient.from_file()
client<jupyter_output><empty_output><jupyter_text>Dataset tracking - OpenFF Gen 2 Opt Set 1 Roche  2020.06.27 01:40 UTC<jupyter_code>dataset = "OpenFF Gen 2 Opt Set 1 Roche"
ds = client.get_collection("OptimizationDataset", dataset)
res = get_unfinished_optimizations(ds, 'default', client)
res
ds.df.index[ds.df.default.index.str.endswith('a')]
ds.df.default[ds.df.default.isnull()]<jupyter_output><empty_output><jupyter_text>What is this??? Perhaps we need to resubmit???<jupyter_code>resm = get_unfinished_optimizations(ds, 'default', client, dropna=True)
resm
restart_optimizations(resm, client)<jupyter_output><empty_output><jupyter_text>2020.06.18 15:30 UTC<jupyter_code>dataset = "OpenFF Gen 2 Opt Set 1 Roche"
ds = client.get_collection("OptimizationDataset", dataset)
resm = get_unfinished_optimizations(ds, 'default', client)
resm
ds.df.index[ds.df.default.index.str.endswith('a')]
ds.df.default[ds.df.default.isnull()]<jupyter_output><empty_output><jupyter_text>What is this??? Perhaps we need to resubmit???<jupyter_code>resm = get_unfinished_optimizations(ds, 'default', client, dropna=True)
resm
restart_optimizations(resm, client)<jupyter_output><empty_output><jupyter_text>2020.06.18 14:30 UTC<jupyter_code>dataset = "OpenFF Gen 2 Opt Set 1 Roche"
ds = client.get_collection("OptimizationDataset", dataset)
resm = get_unfinished_optimizations(ds, 'default', client)
resm
ds.df.default[ds.df.default.isnull()]<jupyter_output><empty_output><jupyter_text>What is this??? Perhaps we need to resubmit???<jupyter_code>resm = get_unfinished_optimizations(ds, 'default', client, dropna=True)
resm
restart_optimizations(resm, client)<jupyter_output><empty_output><jupyter_text>2020.06.11 19:30 UTC<jupyter_code>dataset = "OpenFF Gen 2 Opt Set 1 Roche"
ds = client.get_collection("OptimizationDataset", dataset)
resm = get_unfinished_optimizations(ds, 'default', client)
resm
ds.df.default[ds.df.default.isnull()]<jupyter_output><empty_output><jupyter_text>What is this??? Perhaps we need to resubmit???<jupyter_code>resm = get_unfinished_optimizations(ds, 'default', client, dropna=True)
resm
restart_optimizations(resm, client)<jupyter_output><empty_output><jupyter_text>Dataset tracking - OpenFF Gen 2 Opt Set 2 Coverage  2020.06.27 01:42<jupyter_code>dataset = "OpenFF Gen 2 Opt Set 2 Coverage"
ds = client.get_collection("OptimizationDataset", dataset)
resm = get_unfinished_optimizations(ds, 'default', client, dropna=True)
resm
incomplete = (opt.dict() for opt in resm if opt.status == 'INCOMPLETE')
next(incomplete)
count_unique_optimization_error_messages(resm, full=True, pretty_print=True)
restart_optimizations(resm, client)<jupyter_output>OptimizationRecord(id='18433828', status='ERROR')
OptimizationRecord(id='18433929', status='ERROR')
OptimizationRecord(id='18433930', status='ERROR')
OptimizationRecord(id='18433944', status='ERROR')
OptimizationRecord(id='18433984', status='ERROR')
OptimizationRecord(id='18433988', status='ERROR')
OptimizationRecord(id='18434010', status='ERROR')
OptimizationRecord(id='18434013', status='ERROR')
OptimizationRecord(id='18434015', status='ERROR')
OptimizationRecord(id='18434075', status='ERROR')
OptimizationRecord(id='18434085', status='ERROR')
OptimizationRecord(id='18434088', status='ERROR')
OptimizationRecord(id='18434121', status='ERROR')<jupyter_text>Just INCOMPLETEs now.  2020.06.18 14:35<jupyter_code>dataset = "OpenFF Gen 2 Opt Set 2 Coverage"
ds = client.get_collection("OptimizationDataset", dataset)
resm = get_unfinished_optimizations(ds, 'default', client, dropna=True)
resm
count_unique_optimization_error_messages(resm, full=True, pretty_print=True)
restart_optimizations(resm, client)<jupyter_output><empty_output><jupyter_text>Just INCOMPLETEs now.<jupyter_code>incomplete = (opt.dict() for opt in resm if opt.status == 'INCOMPLETE')
next(incomplete)<jupyter_output><empty_output><jupyter_text>All INCOMPLETEs about a week old, since we submitted last.  2020.06.11 22:00<jupyter_code>dataset = "OpenFF Gen 2 Opt Set 2 Coverage"
ds = client.get_collection("OptimizationDataset", dataset)
resm = get_unfinished_optimizations(ds, 'default', client, dropna=True)
resm
count_unique_optimization_error_messages(resm, full=True, pretty_print=True)
restart_optimizations(resm, client)<jupyter_output>OptimizationRecord(id='18433828', status='ERROR')
OptimizationRecord(id='18433929', status='ERROR')
OptimizationRecord(id='18433930', status='ERROR')
OptimizationRecord(id='18433937', status='ERROR')
OptimizationRecord(id='18433944', status='ERROR')
OptimizationRecord(id='18433984', status='ERROR')
OptimizationRecord(id='18433988', status='ERROR')
OptimizationRecord(id='18434010', status='ERROR')
OptimizationRecord(id='18434013', status='ERROR')
OptimizationRecord(id='18434015', status='ERROR')
OptimizationRecord(id='18434075', status='ERROR')
OptimizationRecord(id='18434085', status='ERROR')
OptimizationRecord(id='18434088', status='ERROR')
OptimizationRecord(id='18434121', status='ERROR')<jupyter_text>Dataset tracking - OpenFF Gen 2 Opt Set 3 Pfizer Discrepancy  2020.06.18 14:40<jupyter_code>dataset = "OpenFF Gen 2 Opt Set 3 Pfizer Discrepancy"
ds = client.get_collection("OptimizationDataset", dataset)
resm = get_unfinished_optimizations(ds, 'default', client, dropna=True)
resm
ds.status('default')
count_unique_optimization_error_messages(resm, full=True, pretty_print=True)
restart_optimizations(resm, client)<jupyter_output><empty_output><jupyter_text>Great!  2020.06.11 22:00<jupyter_code>dataset = "OpenFF Gen 2 Opt Set 3 Pfizer Discrepancy"
ds = client.get_collection("OptimizationDataset", dataset)
resm = get_unfinished_optimizations(ds, 'default', client, dropna=True)
resm
count_unique_optimization_error_messages(resm, full=True, pretty_print=True)
restart_optimizations(resm, client)<jupyter_output><empty_output><jupyter_text>Dataset tracking - OpenFF Gen 2 Opt Set 4 eMolecules Discrepancy  2020.06.27 01:48 UTC<jupyter_code>dataset = "OpenFF Gen 2 Opt Set 4 eMolecules Discrepancy"
ds = client.get_collection("OptimizationDataset", dataset)
resm = get_unfinished_optimizations(ds, 'default', client, dropna=True)
resm
incomplete = (opt.dict() for opt in resm if opt.status == 'INCOMPLETE')
next(incomplete)
count_unique_optimization_error_messages(resm, full=True, pretty_print=True)
restart_optimizations(resm, client)<jupyter_output>OptimizationRecord(id='18437465', status='ERROR')
OptimizationRecord(id='18436178', status='ERROR')<jupyter_text>2020.06.18 14:45<jupyter_code>dataset = "OpenFF Gen 2 Opt Set 4 eMolecules Discrepancy"
ds = client.get_collection("OptimizationDataset", dataset)
resm = get_unfinished_optimizations(ds, 'default', client, dropna=True)
resm
count_unique_optimization_error_messages(resm, full=True, pretty_print=True)
restart_optimizations(resm, client)
incomplete = (opt.dict() for opt in resm if opt.status == 'INCOMPLETE')
next(incomplete)<jupyter_output><empty_output><jupyter_text>INCOMPLETEs running on PRP and UCI. 1 week old.   2020.06.11 22:00<jupyter_code>dataset = "OpenFF Gen 2 Opt Set 4 eMolecules Discrepancy"
ds = client.get_collection("OptimizationDataset", dataset)
resm = get_unfinished_optimizations(ds, 'default', client, dropna=True)
resm
count_unique_optimization_error_messages(resm, full=True, pretty_print=True)
restart_optimizations(resm, client)<jupyter_output>OptimizationRecord(id='18437465', status='ERROR')
OptimizationRecord(id='18436178', status='ERROR')
OptimizationRecord(id='18437470', status='ERROR')
OptimizationRecord(id='18438248', status='ERROR')
OptimizationRecord(id='18438079', status='ERROR')<jupyter_text>----  Previous work<jupyter_code>dataset = "OpenFF Gen 2 Opt Set 4 eMolecules Discrepancy"
ds = client.get_collection("OptimizationDataset", dataset)
ds.list_specifications()
ds.status("default")
res = get_failed_optimizations(ds, 'default', client)
for i in sorted([i.id for i in res if i.status != 'COMPLETE']):
    print(i)
errors = (i.get_error().error_message for i in res if i.status == 'ERROR')
print(next(errors))<jupyter_output>geomeTRIC run_json error:
Traceback (most recent call last):
  File "/opt/conda/envs/qcfractal/lib/python3.6/site-packages/geometric/run_json.py", line 214, in geometric_run_json
    geometric.optimize.Optimize(coords, M, IC, engine, None, params)
  File "/opt/conda/envs/qcfractal/lib/python3.6/site-packages/geometric/optimize.py", line 1331, in Optimize
    return optimizer.optimizeGeometry()
  File "/opt/conda/envs/qcfractal/lib/python3.6/site-packages/geometric/optimize.py", line 1301, in optimizeGeometry
    raise GeomOptNotConvergedError("Optimizer.optimizeGeometry() failed to converge.")
geometric.errors.GeomOptNotConvergedError: Optimizer.optimizeGeometry() failed to converge.<jupyter_text>For convergence errors, if we can look at the initial and final molecules in NGLview, can give indications as to why:<jupyter_code>opt = res[0]
opt.get_initial_molecule()
opt.get_final_molecule()<jupyter_output><empty_output><jupyter_text>All of these are convergence failures.  Dataset tracking - OpenFF Gen 2 Opt Set 5 Bayer  2020.06.27 01:51 UTC<jupyter_code>dataset = "OpenFF Gen 2 Opt Set 5 Bayer"
ds = client.get_collection("OptimizationDataset", dataset)
resm = get_unfinished_optimizations(ds, 'default', client, dropna=True)
resm
len(resm)
incomplete = (opt.dict() for opt in resm if opt.status == 'INCOMPLETE')
next(incomplete)<jupyter_output><empty_output><jupyter_text>PRP INCOMPLETE only.<jupyter_code>count_unique_optimization_error_messages(resm, full=True, pretty_print=True)
restart_optimizations(resm, client)<jupyter_output>OptimizationRecord(id='19094237', status='ERROR')
OptimizationRecord(id='19094393', status='ERROR')
OptimizationRecord(id='19094394', status='ERROR')
OptimizationRecord(id='19094400', status='ERROR')
OptimizationRecord(id='19094836', status='ERROR')
OptimizationRecord(id='19095157', status='ERROR')
OptimizationRecord(id='19095217', status='ERROR')
OptimizationRecord(id='19095261', status='ERROR')
OptimizationRecord(id='19095387', status='ERROR')
OptimizationRecord(id='19095395', status='ERROR')
OptimizationRecord(id='19095607', status='ERROR')
OptimizationRecord(id='19095795', status='ERROR')
OptimizationRecord(id='19095808', status='ERROR')
OptimizationRecord(id='19094233', status='ERROR')
OptimizationRecord(id='19094485', status='ERROR')
OptimizationRecord(id='19095160', status='ERROR')
OptimizationRecord(id='19095389', status='ERROR')
OptimizationRecord(id='19095797', status='ERROR')
OptimizationRecord(id='19095805', status='ERROR')
OptimizationRecord(id='19095886', status='ERROR')
[...]<jupyter_text>2020.06.18 14:45<jupyter_code>dataset = "OpenFF Gen 2 Opt Set 5 Bayer"
ds = client.get_collection("OptimizationDataset", dataset)
resm = get_unfinished_optimizations(ds, 'default', client, dropna=True)
resm
len(resm)
incomplete = (opt.dict() for opt in resm if opt.status == 'INCOMPLETE')
next(incomplete)<jupyter_output><empty_output><jupyter_text>PRP, UCI, and one Galileo INCOMPLETE. This is a major problem.<jupyter_code>count_unique_optimization_error_messages(resm, full=True, pretty_print=True)
restart_optimizations(resm, client)<jupyter_output><empty_output><jupyter_text>2020.06.11 22:00<jupyter_code>dataset = "OpenFF Gen 2 Opt Set 5 Bayer"
ds = client.get_collection("OptimizationDataset", dataset)
resm = get_unfinished_optimizations(ds, 'default', client, dropna=True)
resm
incomplete = (opt.dict() for opt in resm if opt.status == 'INCOMPLETE')
next(incomplete)<jupyter_output><empty_output><jupyter_text>PRP unfinished. Perhaps when the manager gets kicked this will get released?<jupyter_code>count_unique_optimization_error_messages(resm, full=True, pretty_print=True)
restart_optimizations(resm, client)<jupyter_output>OptimizationRecord(id='19094237', status='ERROR')
OptimizationRecord(id='19094393', status='ERROR')
OptimizationRecord(id='19094394', status='ERROR')
OptimizationRecord(id='19094397', status='ERROR')
OptimizationRecord(id='19094836', status='ERROR')
OptimizationRecord(id='19095160', status='ERROR')
OptimizationRecord(id='19095389', status='ERROR')
OptimizationRecord(id='19095795', status='ERROR')
OptimizationRecord(id='19095797', status='ERROR')
OptimizationRecord(id='19095886', status='ERROR')
OptimizationRecord(id='19095887', status='ERROR')
OptimizationRecord(id='19095894', status='ERROR')
OptimizationRecord(id='19094233', status='ERROR')
OptimizationRecord(id='19094400', status='ERROR')
OptimizationRecord(id='19094485', status='ERROR')
OptimizationRecord(id='19095157', status='ERROR')
OptimizationRecord(id='19095158', status='ERROR')
OptimizationRecord(id='19095217', status='ERROR')
OptimizationRecord(id='19095261', status='ERROR')
OptimizationRecord(id='19095377', status='ERROR')
[...]
{'fraction_non_alphanumeric': 0.13051400804798421, 'fraction_numerical': 0.06332093235137803, 'mean_word_length': 10.584872471416007, 'pattern_counts': {'":': 0, '<': 102, '<?xml version=': 0, '>': 102, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '27', 'dir': 'jupyter-structured-clean-dedup', 'id': '605120', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 5030, 'n_tokens_neox': 4403, 'n_words': 844, 'path': 'management/Management - Gen 2 Opt.ipynb', 'repo_name': 'hyejang/qca-dataset-submission', 'size': '13171'}