<jupyter_start><jupyter_text>Dataset<jupyter_code>from SCFInitialGuess.utilities.dataset import extract_triu_batch, AbstractDataset
from sklearn.model_selection import train_test_split

#data_path = "../thesis/dataset/EthenT/"
#postfix = "EthenT"
#dim = 72
#N_ELECTRONS = 16
#basis = "6-311++g**"
data_path = "../thesis/dataset/TSmall/"
postfix = "TSmall"
dim = 70
N_ELECTRONS = 30
basis = "3-21++g*"

#data_path = "../../../butadien/data/"
#postfix = ""
#dim = 26


def split(x, y, ind):
    return x[:ind], y[:ind], x[ind:], y[ind:]

#[S, P] = np.load()
S = np.load(join(data_path, "S" + postfix + ".npy"))
P = np.load(join(data_path, "P" + postfix + ".npy"))

index = np.load(join(data_path, "index" + postfix + ".npy"))


#ind = int(0.8 * len(index))
ind = 0

molecules = np.load(join(data_path, "molecules" + postfix + ".npy"))
molecules = (molecules[:ind], molecules[ind:])
#molecules = ([], molecules)

s_triu_norm, mu, std = AbstractDataset.normalize(S)

s_train, p_train, s_test, p_test = split(S.reshape(-1, dim, dim), P.reshape(-1, dim, dim), ind)
#s_test = S.reshape(-1, dim, dim)
#p_test = S.reshape(-1, dim, dim)
from SCFInitialGuess.utilities.analysis import make_results_str, measure_all_quantities
from SCFInitialGuess.utilities.dataset import StaticDataset

dataset = StaticDataset(
    train=(s_train, p_train),
    #train=(None, None),
    validation=(None, None),
    test=(s_test, p_test),
    mu=mu,
    std=std
)
def f(x,y):
    return 4*x + 6 * y
f(13,3)
from pyscf.scf import hf
h_test = [hf.get_hcore(mol.get_pyscf_molecule()) for mol in molecules[1]]<jupyter_output><empty_output><jupyter_text>Utilities<jupyter_code>from SCFInitialGuess.utilities.dataset import reconstruct_from_triu

def embedd(x, y):
    p = x.copy()
    p[mask] = (y.copy())[mask]
    return p

def embedd_batch(p_batch):
    p_embedded = []
    for (p_guess, p_conv) in zip(p_batch, p_test):
        p_embedded.append(embedd(p_guess, p_conv))
    return np.array(p_embedded)

from SCFInitialGuess.utilities.constants import number_of_basis_functions as N_BASIS

mol = molecules[1][0]

mask = np.zeros((dim, dim))

current_dim = 0
for atom in mol.species:
    
    # calculate block range
    index_start = current_dim
    current_dim += N_BASIS[basis][atom] 
    index_end = current_dim
    
    # calculate logical vector
    L = np.arange(dim)
    L = np.logical_and(index_start <= L, L < index_end)
    
    m = np.logical_and.outer(L, L)
    mask = np.logical_or(mask, m)
    
#mask
import seaborn as sns

sns.heatmap(mask.astype("int"))
np.mean(np.abs(p_test.flatten() - embedd_batch(p_test).flatten()))
from pyscf.scf import hf
def fock_from_density_batch(p_batch):
    f = []
    for p, s, h, mol in zip(p_batch, s_test, h_test, molecules[1]):
        f.append(hf.get_fock(None, h1e=h, s1e=s, vhf=hf.get_veff(mol=mol.get_pyscf_molecule(), dm=p), dm=p))
    return np.array(f)
from SCFInitialGuess.utilities.dataset import density_from_fock

def density_from_fock_batch(f_batch):
    p = []
    for (s, f, mol) in zip(s_test, f_batch, molecules[1]):
        p.append(density_from_fock(f, s, mol.get_pyscf_molecule()))
    return np.array(p)<jupyter_output><empty_output><jupyter_text>GWH<jupyter_code>from pyscf.scf import hf

p_gwh = np.array([ 
    hf.init_guess_by_wolfsberg_helmholtz(mol.get_pyscf_molecule()) for mol in molecules[1]
]).astype("float64")
from SCFInitialGuess.utilities.analysis import make_results_str, measure_all_quantities, mf_initializer

print(make_results_str(measure_all_quantities(
    p_gwh,
    dataset,
    molecules[1],
    N_ELECTRONS,
    mf_initializer,
    dim,
    is_triu=False,
    is_dataset_triu=False,
    s=S[ind:]
)))<jupyter_output>[ ] 2019-01-07 12:58:22: Iteration calculation: 0
[ ] 2019-01-07 12:58:24: Iteration calculation: 1
[ ] 2019-01-07 12:58:26: Iteration calculation: 2
[ ] 2019-01-07 12:58:29: Iteration calculation: 3
[ ] 2019-01-07 12:58:31: Iteration calculation: 4
[ ] 2019-01-07 12:58:33: Iteration calculation: 5
[ ] 2019-01-07 12:58:36: Iteration calculation: 6
[ ] 2019-01-07 12:58:38: Iteration calculation: 7
[ ] 2019-01-07 12:58:40: Iteration calculation: 8
[ ] 2019-01-07 12:58:42: Iteration calculation: 9
[ ] 2019-01-07 12:58:44: Iteration calculation: 10
[ ] 2019-01-07 12:58:46: Iteration calculation: 11
[ ] 2019-01-07 12:58:48: Iteration calculation: 12
[ ] 2019-01-07 12:58:50: Iteration calculation: 13
[ ] 2019-01-07 12:58:52: Iteration calculation: 14
[ ] 2019-01-07 12:58:54: Iteration calculation: 15
[ ] 2019-01-07 12:58:56: Iteration calculation: 16
[ ] 2019-01-07 12:58:57: Iteration calculation: 17
[ ] 2019-01-07 12:58:58: Iteration calculation: 18
[ ] 2019-01-07 12:58:59: Iteration calcul[...]<jupyter_text>Embedded GWH<jupyter_code>p_embedded_gwh = embedd_batch(p_gwh)
from SCFInitialGuess.utilities.analysis import mf_initializer as mf_initializer

print(make_results_str(measure_all_quantities(
    p_embedded_gwh,
    dataset,
    molecules[1],
    N_ELECTRONS,
    mf_initializer,
    dim,
    is_triu=False,
    is_dataset_triu=False,
    s=S[ind:]
)))
51/1001<jupyter_output><empty_output><jupyter_text>Embedded GWH + 1 Iteration<jupyter_code>f_embedded_gwh = fock_from_density_batch(p_embedded_gwh)
p_embedded_gwh_test = density_from_fock_batch(f_embedded_gwh)
from SCFInitialGuess.utilities.analysis import mf_initializer as mf_initializer

print(make_results_str(measure_all_quantities(
    p_embedded_gwh_test,
    dataset,
    molecules[1],
    N_ELECTRONS,
    mf_initializer,
    dim,
    is_triu=False,
    is_dataset_triu=False,
    s=S[ind:]
)))<jupyter_output>[ ] 2019-01-07 14:24:40: Iteration calculation: 0
[ ] 2019-01-07 14:24:42: Iteration calculation: 1
[ ] 2019-01-07 14:24:44: Iteration calculation: 2
[ ] 2019-01-07 14:24:47: Iteration calculation: 3
[ ] 2019-01-07 14:24:49: Iteration calculation: 4
[ ] 2019-01-07 14:24:51: Iteration calculation: 5
[ ] 2019-01-07 14:24:53: Iteration calculation: 6
[ ] 2019-01-07 14:24:55: Iteration calculation: 7
[ ] 2019-01-07 14:24:57: Iteration calculation: 8
[ ] 2019-01-07 14:24:59: Iteration calculation: 9
[ ] 2019-01-07 14:25:01: Iteration calculation: 10
[ ] 2019-01-07 14:25:03: Iteration calculation: 11
[ ] 2019-01-07 14:25:05: Iteration calculation: 12
[ ] 2019-01-07 14:25:07: Iteration calculation: 13
[ ] 2019-01-07 14:25:10: Iteration calculation: 14
[ ] 2019-01-07 14:25:12: Iteration calculation: 15
[ ] 2019-01-07 14:25:14: Iteration calculation: 16
[ ] 2019-01-07 14:25:15: Iteration calculation: 17
[ ] 2019-01-07 14:25:16: Iteration calculation: 18
[ ] 2019-01-07 14:25:17: Iteration calcul[...]<jupyter_text>SAD<jupyter_code>from pyscf.scf import hf

p_sad = np.array([ 
    hf.init_guess_by_atom(mol.get_pyscf_molecule()) for mol in molecules[1]
]).astype("float64")
sns.heatmap((abs(p_sad[0]) < 1e-12).astype("int"))
sns.heatmap((abs(p_gwh[0]) < 0.05).astype("int"))
sns.heatmap((abs(p_test[30]) < 0.05).astype("int"))
from SCFInitialGuess.utilities.analysis import make_results_str, measure_all_quantities, mf_initializer

print(make_results_str(measure_all_quantities(
    p_sad,
    dataset,
    molecules[1],
    N_ELECTRONS,
    mf_initializer,
    dim,
    is_triu=False,
    is_dataset_triu=False,
    s=S[ind:]
)))
47/1001<jupyter_output><empty_output><jupyter_text>Embedded zeros<jupyter_code>p_embedded_zeros = embedd_batch(np.zeros(p_test.shape))
sns.heatmap((abs(p_embedded_zeros[0]) < 0.05).astype("int"))
from SCFInitialGuess.utilities.analysis import mf_initializer as mf_initializer

print(make_results_str(measure_all_quantities(
    p_embedded_zeros,
    dataset,
    molecules[1],
    N_ELECTRONS,
    mf_initializer,
    dim,
    is_triu=False,
    is_dataset_triu=False,
    s=S[ind:]
)))<jupyter_output>[ ] 2019-01-07 15:34:40: Iteration calculation: 0
[ ] 2019-01-07 15:34:42: Iteration calculation: 1
[ ] 2019-01-07 15:34:44: Iteration calculation: 2
[ ] 2019-01-07 15:34:46: Iteration calculation: 3
[ ] 2019-01-07 15:34:48: Iteration calculation: 4
[ ] 2019-01-07 15:34:50: Iteration calculation: 5
[ ] 2019-01-07 15:34:52: Iteration calculation: 6
[ ] 2019-01-07 15:34:53: Iteration calculation: 7
[ ] 2019-01-07 15:34:54: Iteration calculation: 8
[ ] 2019-01-07 15:34:56: Iteration calculation: 9
[ ] 2019-01-07 15:34:57: Iteration calculation: 10
[ ] 2019-01-07 15:34:58: Iteration calculation: 11
[ ] 2019-01-07 15:34:59: Iteration calculation: 12
[ ] 2019-01-07 15:35:00: Iteration calculation: 13
[ ] 2019-01-07 15:35:01: Iteration calculation: 14
[ ] 2019-01-07 15:35:02: Iteration calculation: 15
[ ] 2019-01-07 15:35:03: Iteration calculation: 16
[ ] 2019-01-07 15:35:03: Iteration calculation: 17
[ ] 2019-01-07 15:35:04: Iteration calculation: 18
[ ] 2019-01-07 15:35:06: Iteration calcul[...]<jupyter_text>Embedded zeros + 1 Iteration?<jupyter_code>f_embedded_zeros = fock_from_density_batch(p_embedded_zeros)
p_embedded_zeros_test = density_from_fock_batch(f_embedded_zeros,)
from SCFInitialGuess.utilities.analysis import mf_initializer, make_results_str, measure_all_quantities

print(make_results_str(measure_all_quantities(
    p_embedded_zeros_test,
    dataset,
    molecules[1],
    N_ELECTRONS,
    mf_initializer,
    dim,
    is_triu=False,
    is_dataset_triu=False,
    s=S[ind:]
)))<jupyter_output>[ ] 2019-01-07 16:16:18: Iteration calculation: 0
[ ] 2019-01-07 16:16:20: Iteration calculation: 1
[ ] 2019-01-07 16:16:22: Iteration calculation: 2
[ ] 2019-01-07 16:16:24: Iteration calculation: 3
[ ] 2019-01-07 16:16:26: Iteration calculation: 4
[ ] 2019-01-07 16:16:28: Iteration calculation: 5
[ ] 2019-01-07 16:16:29: Iteration calculation: 6
[ ] 2019-01-07 16:16:31: Iteration calculation: 7
[ ] 2019-01-07 16:16:32: Iteration calculation: 8
[ ] 2019-01-07 16:16:34: Iteration calculation: 9
[ ] 2019-01-07 16:16:35: Iteration calculation: 10
[ ] 2019-01-07 16:16:36: Iteration calculation: 11
[ ] 2019-01-07 16:16:37: Iteration calculation: 12
[ ] 2019-01-07 16:16:38: Iteration calculation: 13
[ ] 2019-01-07 16:16:39: Iteration calculation: 14
[ ] 2019-01-07 16:16:39: Iteration calculation: 15
[ ] 2019-01-07 16:16:40: Iteration calculation: 16
[ ] 2019-01-07 16:16:41: Iteration calculation: 17
[ ] 2019-01-07 16:16:42: Iteration calculation: 18
[ ] 2019-01-07 16:16:43: Iteration calcul[...]<jupyter_text><jupyter_code>77/1001<jupyter_output><empty_output><jupyter_text>Embedded GWH w/ Self-Overlap<jupyter_code>from SCFInitialGuess.utilities.constants import number_of_basis_functions as N_BASIS

mol = molecules[1][0]

mask_self_overlap = np.zeros((dim, dim))

current_dim_i = 0
for i, atom_i in enumerate(mol.species):
    
    
    # calculate block range
    index_start_i = current_dim_i
    current_dim_i += N_BASIS[basis][atom_i] 
    index_end_i = current_dim_i
    
    # calculate logical vector
    L_i = np.arange(dim)
    L_i = np.logical_and(index_start_i <= L_i, L_i < index_end_i)
    
    current_dim_j = 0
    for j, atom_j in enumerate(mol.species):
        

        #print(str(i) + ", " + str(j))    
        #print(str(atom_i) + ", " + str(atom_j))
        
        # calculate block range
        index_start_j = current_dim_j
        current_dim_j += N_BASIS[basis][atom_j] 
        index_end_j = current_dim_j

        if i == j:
            continue
        
        if atom_i == atom_j:
            # calculate logical vector
            L_j = np.arange(dim)
            L_j = np.logical_and(index_start_j <= L_j, L_j < index_end_j)


            m = np.logical_and.outer(L_i, L_j)
            
            mask_self_overlap = np.logical_or(mask_self_overlap, m)
    
#mask
import seaborn as sns

sns.heatmap(mask_self_overlap.astype("int"))
from SCFInitialGuess.utilities.dataset import reconstruct_from_triu

def embedd_self_ovlp(x, y):
    p = x.copy()
    p[mask_self_overlap] = (y.copy())[mask_self_overlap]
    return p

def embedd_batch_self_ovlp(p_batch):
    p_embedded = []
    for (p_guess, p_conv) in zip(p_batch, p_test):
        p_embedded.append(embedd_self_ovlp(p_guess, p_conv))
    return np.array(p_embedded)

p_embedded_gwh_self_ovlp = embedd_batch_self_ovlp(p_embedded_gwh)
sns.heatmap(p_embedded_gwh_self_ovlp[0] - p_test[0], square=True)
from SCFInitialGuess.utilities.analysis import mf_initializer, make_results_str, measure_all_quantities

print(make_results_str(measure_all_quantities(
    density_from_fock_batch(fock_from_density_batch(p_embedded_gwh_self_ovlp)),
    dataset,
    molecules[1],
    N_ELECTRONS,
    mf_initializer,
    dim,
    is_triu=False,
    is_dataset_triu=False,
    s=S[ind:]
)))
6/(len(p_test))<jupyter_output><empty_output><jupyter_text>Embedded GWH w/ OFF overlap<jupyter_code>from SCFInitialGuess.utilities.constants import number_of_basis_functions as N_BASIS

mol = molecules[1][0]

mask_off_overlap = np.zeros((dim, dim))

current_dim_i = 0
for i, atom_i in enumerate(mol.species):
    
    
    # calculate block range
    index_start_i = current_dim_i
    current_dim_i += N_BASIS[basis][atom_i] 
    index_end_i = current_dim_i
    
    # calculate logical vector
    L_i = np.arange(dim)
    L_i = np.logical_and(index_start_i <= L_i, L_i < index_end_i)
    
    current_dim_j = 0
    for j, atom_j in enumerate(mol.species):
        

        #print(str(i) + ", " + str(j))    
        #print(str(atom_i) + ", " + str(atom_j))
        
        # calculate block range
        index_start_j = current_dim_j
        current_dim_j += N_BASIS[basis][atom_j] 
        index_end_j = current_dim_j

        if i == j:
            continue
        
        if atom_i != atom_j:
            # calculate logical vector
            L_j = np.arange(dim)
            L_j = np.logical_and(index_start_j <= L_j, L_j < index_end_j)


            m = np.logical_and.outer(L_i, L_j)
            
            mask_off_overlap = np.logical_or(mask_off_overlap, m)
    
#mask
import seaborn as sns

sns.heatmap(mask_off_overlap.astype("int"))
from SCFInitialGuess.utilities.dataset import reconstruct_from_triu

def embedd_off_ovlp(x, y):
    p = x.copy()
    p[mask_off_overlap] = (y.copy())[mask_off_overlap]
    return p

def embedd_batch_off_ovlp(p_batch):
    p_embedded = []
    for (p_guess, p_conv) in zip(p_batch, p_test):
        p_embedded.append(embedd_off_ovlp(p_guess, p_conv))
    return np.array(p_embedded)

p_embedded_gwh_off_ovlp = embedd_batch_off_ovlp(p_embedded_gwh)
sns.heatmap(p_embedded_gwh_off_ovlp[0] - p_test[0], square=True)
from SCFInitialGuess.utilities.analysis import mf_initializer, make_results_str, measure_all_quantities

print(make_results_str(measure_all_quantities(
    density_from_fock_batch(fock_from_density_batch(p_embedded_gwh_off_ovlp)),
    dataset,
    molecules[1],
    N_ELECTRONS,
    mf_initializer,
    dim,
    is_triu=False,
    is_dataset_triu=False,
    s=S[ind:]
)))
7/len(p_test)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12931792063813966, 'fraction_numerical': 0.09017778679104982, 'mean_word_length': 4.27415329768271, 'pattern_counts': {'":': 0, '<': 55, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '41808', 'license': 'MIT', 'n_tokens_mistral': 6909, 'n_tokens_neox': 5783, 'n_words': 1074, 'path': 'notebooks/Construction/ConstructionSchemes_Density_TSmall.ipynb', 'repo_name': 'jcartus/SCFInitialGuess', 'size': '14793'}