<jupyter_start><jupyter_text>Drastic Data Handling to split or refine parameters for better decomposition<jupyter_code>import time
import numpy as np
import tensorly as tl
from tensorly.decomposition import parafac
from tensorly.decomposition.candecomp_parafac import initialize_factors, unfolding_dot_khatri_rao, KruskalTensor
# for sample video
from cv2 import VideoWriter, VideoWriter_fourcc

def make_video(tensor, filename):
    start = time.time()
    height = tensor.shape[1]
    width = tensor.shape[2]
    FPS = 24

    fourcc = VideoWriter_fourcc(*'MP42')
    video = VideoWriter(filename, fourcc, float(FPS), (width, height))

    for frame in tensor:
        video.write(np.uint8(frame))
    video.release()
    print('created', filename, time.time()-start)
import math
class Welford(object):
    def __init__(self,lst=None):
        self.k = 0
        self.M = 0
        self.S = 0
        
        self.__call__(lst)
    
    def update(self,x):
        if x is None:
            return
        self.k += 1
        newM = self.M + (x - self.M)*1./self.k
        newS = self.S + (x - self.M)*(x - newM)
        self.M, self.S = newM, newS

    def consume(self,lst):
        lst = iter(lst)
        for x in lst:
            self.update(x)
    
    def __call__(self,x):
        if hasattr(x,"__iter__"):
            self.consume(x)
        else:
            self.update(x)
            
    @property
    def mean(self):
        return self.M
    @property
    def meanfull(self):
        return self.mean, self.std/math.sqrt(self.k)
    @property
    def std(self):
        if self.k==1:
            return 0
        return math.sqrt(self.S/(self.k-1))
    def __repr__(self):
        return "<Welford: {} +- {}>".format(self.mean, self.std)
def construct_tensor(factors):
    weights = tl.ones(factors[0].shape[1])
    est_tensor = tl.kruskal_to_tensor((weights, factors))
    return est_tensor
    
def print_tensor(X, n_digit=1):
    print(np.round(X, n_digit))
    
def compare_tensors(A, B):
    error_norm = tl.norm(A - B)
    print('||A-B||:', error_norm)
    return error_norm
    
def create_tensor_stream(X, start_to_stream, batch_sizes):
    total_batch_size = np.sum(batch_sizes)
    if X.shape[0] != start_to_stream + total_batch_size:
        raise ValueError('Total batch size should be the size of streaming part of the tensor.')
    
    X_stream = [X[:start_to_stream]]
    batch_start = start_to_stream
    for batch_size in batch_sizes:
        batch_end = batch_start + batch_size
        X_stream.append(X[batch_start:batch_end])
        batch_start = batch_end
    return np.asarray(X_stream)
    
def get_KhatriRao(factors):
    n_dim = len(factors)
    lefts = [factors[n_dim-1]]
    rights = [factors[0]]
    if n_dim > 2:
        for mode in range(1, n_dim-1):
            lefts.append(tl.tenalg.khatri_rao((lefts[mode-1], factors[n_dim-mode-1])))
            rights.append(tl.tenalg.khatri_rao((factors[mode], rights[mode-1])))
            
    K = lefts.copy()
    K[0] = lefts[n_dim-2]
    K.append(rights[n_dim-2].copy())
    if n_dim > 2:
        for mode in range(1, n_dim-1):
            K[mode] = tl.tenalg.khatri_rao((lefts[n_dim-mode-2], rights[mode-1]))
    return K

def get_KhatriRao_except0(factors):
    n_dim = len(factors)
    lefts = np.empty((n_dim), dtype=object)
    rights = np.empty((n_dim), dtype=object)
    K = np.empty((n_dim), dtype=object)
    
    lefts[1] = factors[n_dim-1]
    rights[1] = factors[1]
    if n_dim > 3:
        for mode in range(2, n_dim-1):
            lefts[mode] = tl.tenalg.khatri_rao((factors[n_dim-mode], lefts[mode-1]))
            rights[mode] = tl.tenalg.khatri_rao((rights[mode-1], factors[mode]))
            
    K[1] = lefts[n_dim-2]
    K[n_dim-1] = rights[n_dim-2]
    if n_dim > 3: 
        for mode in range(2, n_dim-1):
            K[mode] = tl.tenalg.khatri_rao((rights[mode-1], lefts[n_dim-mode-1]))
    return K
    
def get_Hadamard(factors):
    rank = factors[0].shape[1]
    H = tl.tensor(np.ones((rank, rank)))
    for factor in factors:
        H = H * tl.dot(tl.transpose(factor), factor)
    return H<jupyter_output><empty_output><jupyter_text>Online CP<jupyter_code>def online_cp(factors_old, X_old, X_new, rank, P, Q, n_iter=1, mu=1, verbose=False, transformed=False):
    weights = tl.ones(rank)
    if verbose:
        X = tl.tensor(np.concatenate((X_old, X_new)))
    n_dim = tl.ndim(X_old)
    U = factors_old.copy()
    
    if not transformed:
        K = get_KhatriRao_except0(factors_old)
    H = get_Hadamard(factors_old[1:])
        
    for i in range(n_iter):
        # temporal mode for A1
        if not transformed:
            mttkrp = tl.dot(tl.unfold(X_new, 0), tl.tenalg.khatri_rao((U[1], K[1])))
        else:
            # for higher accracy, lower speed
            mttkrp_parts = []
            for r in range(rank):
                component = tl.tenalg.multi_mode_dot(X_new, [f[:, r] for f in U], skip=0)
                mttkrp_parts.append(component)
            mttkrp = np.stack(mttkrp_parts, axis=1)
        
        A1 = tl.transpose(tl.solve(tl.transpose(H), tl.transpose(mttkrp)))

        # non-temporal mode
        for mode in range(1, n_dim):
            
            if not transformed:
                dP = tl.dot(tl.unfold(X_new, mode), tl.tenalg.khatri_rao((A1, K[mode])))
                UTU  = tl.dot(tl.transpose(U[mode]), U[mode])
                dQ = tl.dot(tl.transpose(A1), A1) * H / UTU
                
                U[mode] = tl.transpose(tl.solve(tl.transpose(mu*Q[mode] + dQ), tl.transpose(mu*P[mode] + dP)))
#                 K = updated K due to non-temporal mode change
#                 H = H_mode * tl.dot(tl.transpose(U[mode]), U[mode]) / UTU
                P[mode] = P[mode] + dP
                Q[mode] = Q[mode] + dQ
            else:
                U1 = U.copy()
                U1[0] = A1
                
                H_mode  = H / tl.dot(tl.transpose(U[mode]), U[mode])
                V = (mu * tl.dot(tl.transpose(U[0]), U[0]) + tl.dot(tl.transpose(A1), A1)) * H_mode
                
                mttkrp0 = unfolding_dot_khatri_rao(X_old, (None, U), mode)
                mttkrp1 = unfolding_dot_khatri_rao(X_new, (None, U1), mode)
                
                U[mode] = tl.transpose(tl.solve(tl.transpose(V), tl.transpose(mu*mttkrp0 + mttkrp1)))
                H = H_mode * tl.dot(tl.transpose(U[mode]), U[mode])
                
        # temporal mode for A0
        if transformed:
            mttkrp = unfolding_dot_khatri_rao(X_old, (None, U), 0)
            U[0] = tl.transpose(tl.solve(tl.transpose(H), tl.transpose(mttkrp)))
#         else:
#             V = tl.tensor(np.ones((rank, rank)))
#             W = tl.tensor(np.ones((rank, rank)))
#             for j, factor in enumerate(U):
#                 factor_old = factors_old[j]
#                 if j != 0:
#                     V = V * tl.dot(tl.transpose(factor), factor)
#                     W = W * tl.dot(tl.transpose(factor_old), factor)
#             mttkrp = tl.dot(factors_old[0], W)
#             U[0] = tl.transpose(tl.solve(tl.transpose(V), tl.transpose(mttkrp)))
            
        if verbose:
            U1 = U.copy()
            U1[0] = np.concatenate((U[0], A1))
            X_est = construct_tensor(U1)
            compare_tensors(X, X_est)

    U[0] = np.concatenate((U[0], A1))
    return (KruskalTensor((weights, U)), P, Q)<jupyter_output><empty_output><jupyter_text>DTD<jupyter_code>def dtd(factors_old, X_old, X_new, rank, n_iter=1, mu=1, verbose=False):
    
    weights = tl.ones(rank)
    if verbose:
        X = tl.tensor(np.concatenate((X_old, X_new)))
    n_dim = tl.ndim(X_old)
    U = factors_old.copy()
    
    for i in range(n_iter):
        # temporal mode for A1
        V = tl.tensor(np.ones((rank, rank)))
        for j, factor in enumerate(U):
            if j != 0:
                V = V * tl.dot(tl.transpose(factor), factor)
        mttkrp = unfolding_dot_khatri_rao(X_new, (None, U), 0)
        A1 = tl.transpose(tl.solve(tl.transpose(V), tl.transpose(mttkrp)))

        # non-temporal mode
        for mode in range(1, n_dim):
            U1 = U.copy()
            U1[0] = A1
            V = tl.tensor(np.ones((rank, rank)))
            W = tl.tensor(np.ones((rank, rank)))
            for j, factor in enumerate(U):
                factor_old = factors_old[j]
                if j != mode:
                    W = W * tl.dot(tl.transpose(factor_old), factor)
                    if j == 0:
                        V = V * (mu*tl.dot(tl.transpose(factor), factor) + tl.dot(tl.transpose(A1), A1))
                    else:
                        V = V * tl.dot(tl.transpose(factor), factor)
            mttkrp0 = mu * tl.dot(factors_old[mode], W)
            mttkrp1 = unfolding_dot_khatri_rao(X_new, (None, U1), mode)
            U[mode] = tl.transpose(tl.solve(tl.transpose(V), tl.transpose(mttkrp0 + mttkrp1)))

        # temporal mode for A0
        V = tl.tensor(np.ones((rank, rank)))
        W = tl.tensor(np.ones((rank, rank)))
        for j, factor in enumerate(U):
            factor_old = factors_old[j]
            if j != 0:
                V = V * tl.dot(tl.transpose(factor), factor)
                W = W * tl.dot(tl.transpose(factor_old), factor)
        mttkrp = tl.dot(factors_old[0], W)
        U[0] = tl.transpose(tl.solve(tl.transpose(V), tl.transpose(mttkrp)))

        if verbose:
            U1 = U.copy()
            U1[0] = np.concatenate((U[0], A1))
            X_est = construct_tensor(U1)
            compare_tensors(X, X_est)

    U[0] = np.concatenate((U[0].copy(), A1))
    return KruskalTensor((weights, U))<jupyter_output><empty_output><jupyter_text>Online Tensor Decomposition* `onlinecp`, `transformed_onlinecp`, `dtd`<jupyter_code>from warnings import warn

def get_z_score(x, mean, std):
    if std == 0:
        return 0
    return (x - mean) / std

def online_tensor_decomposition(X_stream, rank, n_iter=1, ul=-1, ll=-1, verbose=False, method='onlinecp'):
    ktensors = []
    verbose_list = []
    if method == 'onlinecp':
        if n_iter > 1:
            warn('Number of iteration in online cp should not exceed 1 ({} to 1).'.format(n_iter))
            n_iter = 1
        onlinecp = True
        transformed = False
    elif method == 'transformed_onlinecp':
        onlinecp = True
        transformed = True
    elif method == 'dtd':
        onlinecp = False
    else:
        raise ValueError('The method does not exist.')        
        
    X_old = X_stream[0]
    n_dim = tl.ndim(X_old)

    start = time.time()
    (weights, factors) = parafac(X_old, rank, init='random')
    print('making init decomposition result:', time.time()-start)
    
    welford = Welford()
    X_est = construct_tensor(factors)
    err_norm = tl.norm(X_old - X_est) / math.sqrt(X_old.shape[0])
    welford(err_norm * 1.5)
    
    if onlinecp:
        start = time.time()
        print('\n >> onlinecp rank-{} n_iter-{} transformed-{}'.format(rank, n_iter, transformed))
        K = get_KhatriRao_except0(factors)
        H = get_Hadamard(factors)

        P = np.empty((n_dim), dtype=object)
        Q = np.empty((n_dim), dtype=object)
    
        for mode in range(1, n_dim):
            P[mode] = tl.dot(tl.unfold(X_old, mode), tl.tenalg.khatri_rao((factors[0], K[mode])))
            Q[mode] = H / tl.dot(tl.transpose(factors[mode]), factors[mode])
        print('init_time:', time.time()-start)
    else:
        print('\n >> dtd rank-{} n_iter-{}'.format(rank, n_iter))
        
    for i, X_new in enumerate(X_stream[1:]):
        
        start = time.time()
        if onlinecp:
            ((weights, factors0), P0, Q0) = online_cp(factors, X_old, X_new, rank, P, Q, n_iter=n_iter, mu=1, verbose=False, transformed=transformed)
        else:
            (weights, factors0) = dtd(factors, X_old, X_new, rank, n_iter=n_iter, mu=1, verbose=False)
        
        U = factors0.copy()
        U[0] = U[0][-X_new.shape[0]-1:-1]
        dX_est = construct_tensor(U)
        
        err_norm = tl.norm(X_new - dX_est) / math.sqrt(X_new.shape[0])
        z_score = get_z_score(err_norm, welford.mean, welford.std)
        
        if ul > 0 and z_score > ul:
            weights = tl.ones(rank)
            ktensors.append(KruskalTensor((weights, factors.copy())))
            print('=== SPLIT({}, {}) ==='.format(z_score, err_norm))
            welford = Welford()
            
            X_old = X_stream[i]

            (weights, factors) = parafac(X_old, rank, init='random')
            elapsed_time = time.time()-start
            print('making init decomposition result:', time.time()-start)
            verbose_list.append([i+1, elapsed_time, err_norm, z_score])

            start = time.time()
            X_est = construct_tensor(factors)
            err_norm = tl.norm(X_old - X_est) / math.sqrt(X_old.shape[0])
            welford(err_norm * 2)

            if onlinecp:
                K = get_KhatriRao_except0(factors)
                H = get_Hadamard(factors)

                P = np.empty((n_dim), dtype=object)
                Q = np.empty((n_dim), dtype=object)

                for mode in range(1, n_dim):
                    P[mode] = tl.dot(tl.unfold(X_old, mode), tl.tenalg.khatri_rao((factors[0], K[mode])))
                    Q[mode] = H / tl.dot(tl.transpose(factors[mode]), factors[mode])
            
            z_score = get_z_score(err_norm, welford.mean, welford.std)
            welford(err_norm)
            elapsed_time = time.time()-start
            print('{}th_iter:'.format(i+1), elapsed_time, err_norm, z_score)
            verbose_list.append([i+1, elapsed_time, err_norm, z_score])
            continue
        elif ll > 0 and z_score > ll:
            print('=== REFINE({}, {}) ==='.format(z_score, err_norm))
            elapsed_time = time.time()-start
            verbose_list.append([i+1, elapsed_time, err_norm, z_score])
            welford = Welford()
            if onlinecp:
                ((weights, factors), P, Q) = online_cp(factors, X_old, X_new, rank, P, Q, n_iter=n_iter, mu=0.1, verbose=False, transformed=transformed)
            else:
                (weights, factors) = dtd(factors, X_old, X_new, rank, n_iter=n_iter, mu=0.1, verbose=False)

            U = factors.copy()
            U[0] = U[0][-X_new.shape[0]-1:-1]
            dX_est = construct_tensor(U)
            err_norm = tl.norm(X_new - dX_est) / math.sqrt(X_new.shape[0])
        else:
            if onlinecp:
                P = P0
                Q = Q0
            factors = factors0
        z_score = get_z_score(err_norm, welford.mean, welford.std)
        welford(err_norm)
        elapsed_time = time.time()-start
        print('{}th_iter:'.format(i+1), elapsed_time, err_norm, z_score)
        verbose_list.append([i+1, elapsed_time, err_norm, z_score])
        
        X_old = np.concatenate((X_old, X_new))
        
        if verbose:
            X_est = construct_tensor(factors)
            compare_tensors(X_old, X_est)
    
    weights = tl.ones(rank)
    ktensors.append(KruskalTensor((weights, factors)))
    return (ktensors, np.asarray(verbose_list))
import matplotlib.pyplot as plt
def execute_online_tensor_decomposition(X_stream, rank=5, n_iter=1, ul=5, ll=3, verbose=False, methods=['onlinecp', 'transformed_onlinecp', 'dtd']):
    
    filename = 'results/rank{}-n_iter{}'.format(rank, n_iter)
    
    for method in methods:
        ktensors, verbose_list = online_tensor_decomposition(X_stream, rank, n_iter, ul, ll, verbose, method)
        X_est = construct_tensor(ktensors[0][1])
        for (weights, factors) in ktensors[1:]:
            X_est = tl.tensor(np.concatenate((X_est, construct_tensor(factors))))
        global_error_norm = compare_tensors(X, X_est)
#         make_video(X_est, '{}-{}.avi'.format(filename, method))
        
        f = open('{}-{}.txt'.format(filename, method),'w')
        np.savetxt(f, np.asarray([global_error_norm]), delimiter="\n")
        np.savetxt(f, np.asarray(verbose_list), delimiter=" ")
        f.close()
        
        plt.figure(1)
        plt.plot(verbose_list[:,0], verbose_list[:,1], label=method, linewidth=0.5)
        plt.figure(2)
        plt.plot(verbose_list[:,0], verbose_list[:,2], label=method, linewidth=0.5)

    plt.figure(1)
    plt.legend(loc='upper left')
    plt.ylabel('running time for a data slice')
    plt.xlabel('# of data slices')
    plt.savefig('{}-{}.png'.format(filename, 'running_time'), dpi=500)
    
    plt.figure(2)
    plt.legend(loc='upper left')
    plt.ylabel('local error norm for a data slice')
    plt.xlabel('# of data slices')
    plt.savefig('{}-{}.png'.format(filename, 'local_error_norm'), dpi=500)
    plt.show()<jupyter_output><empty_output><jupyter_text>Load Sample Video Dataset<jupyter_code>import csv
sample_video = tl.tensor(np.zeros([205, 240, 320, 3], dtype='d'))

for i in range(41):
    start = time.time()
    with open('../Data/sample_video/data/video{}.tensor'.format(i)) as file:
        reader = csv.reader(file, delimiter='\t')    
        for row in reader:
            indices = [[index] for index in np.int64(np.asarray(row[:-1]))-1]
            sample_video[tuple(indices)] = np.double(row[-1])
    print('>> sample_video{} loaded '.format(i), time.time() - start)
make_video(sample_video, './sample_video/original.avi')
X = sample_video
X_stream = create_tensor_stream(X, start_to_stream=5, batch_sizes=np.full((40), 5, dtype=int))

for n_iter in range(1, 6):
    for rank in range(30, 101, 10):
        execute_online_tensor_decomposition(X_stream, rank, n_iter, ul=5, ll=3, verbose=False)<jupyter_output><empty_output><jupyter_text>Load Synthetic Dataset<jupyter_code>import csv
synthetic_data = tl.tensor(np.zeros([1000, 10, 20, 30], dtype='d'))

for i in range(200):
    start = time.time()
    with open('../Data/synthetic_data/data_normal/data{}.tensor'.format(i)) as file:
        reader = csv.reader(file, delimiter='\t')    
        for row in reader:
            indices = [[index] for index in np.int64(np.asarray(row[:-1]))-1]
            synthetic_data[tuple(indices)] = np.double(row[-1])
    print('>> synthetic_data{} loaded '.format(i), time.time() - start)
X = synthetic_data
X_stream = create_tensor_stream(X, start_to_stream=5, batch_sizes=np.full((199), 5, dtype=int))
execute_online_tensor_decomposition(X_stream, rank=10, n_iter=1, ul=1, ll=0.5, verbose=False)<jupyter_output><empty_output><jupyter_text>Load Server Room CFD<jupyter_code>import csv
server_room_cfd = tl.tensor(np.zeros([3000, 3, 3, 34], dtype='d'))

start = time.time()
welford = Welford()
with open('../Data/ServerRoomCFD.tensor') as file:
    reader = csv.reader(file, delimiter='\t')    
    for row in reader:
        indices = np.asarray([index for index in np.int64(np.asarray(row[:-1]))-1])[[3, 0, 1, 2]]
        
        if indices[0] < 3000:
            server_room_cfd[tuple(indices)] = np.double(row[-1])
            welford(np.double(row[-1]))
server_room_cfd = (server_room_cfd - welford.mean) / welford.std
print('>> ServerRoomCFD loaded ', time.time() - start)
X = server_room_cfd
X_stream = create_tensor_stream(X, start_to_stream=10, batch_sizes=np.full((299), 10, dtype=int))
execute_online_tensor_decomposition(X_stream, rank=10, n_iter=3, ul=-1, ll=-1, verbose=False)<jupyter_output><ipython-input-29-b5523a927d4f>:13: UserWarning: Number of iteration in online cp should not exceed 1 (3 to 1).
  warn('Number of iteration in online cp should not exceed 1 ({} to 1).'.format(n_iter))<jupyter_text>Load Stock Data<jupyter_code>import pickle
with open('../Data/index_data.pkl', 'rb') as f:
    index_data = pickle.load(f)
    
index_data
import csv

stock_data = tl.tensor(np.zeros([855, 189, 5], dtype='d'))
start = time.time()
with open('../Data/Stock_.tensor') as file:
    reader = csv.reader(file, delimiter='\t')    
    for row in reader:
        indices = np.asarray([index for index in np.int64(np.asarray(row[:-1]))])[[1, 0, 2]]
        stock_data[tuple(indices)] = np.double(row[-1])
print('>> stock_data loaded ', time.time() - start)
X = stock_data
X_stream = create_tensor_stream(X, start_to_stream=5, batch_sizes=np.full((170), 5, dtype=int))

execute_online_tensor_decomposition(X_stream, rank=3, n_iter=1, ul=5, ll=3, verbose=False, methods=['onlinecp'])
# execute_online_tensor_decomposition(X_stream, rank, n_iter, ul=5, ll=3, verbose=False, methods=['onlinecp', 'dtd'])
850/5
import csv
stock_data = tl.tensor(np.zeros([3000, 200, 5], dtype='d'))

start = time.time()
with open('../Data/Stock.tensor') as file:
    reader = csv.reader(file, delimiter='\t')    
    for row in reader:
        indices = np.asarray([index for index in np.int64(np.asarray(row[:-1]))])[[1, 0, 2]]
        if indices[0] < 3000:
            stock_data[tuple(indices)] = np.double(row[-1])
print('>> stock_data loaded ', time.time() - start)
X = stock_data
X_stream = create_tensor_stream(X, start_to_stream=10, batch_sizes=np.full((299), 10, dtype=int))

for n_iter in range(5, 55, 5):
    for rank in range(5, 15, 5):
        execute_online_tensor_decomposition(X_stream, rank, n_iter, ul=5, ll=3, verbose=False)
execute_online_tensor_decomposition(X_stream, rank, n_iter, ul=5, ll=3, verbose=False)
# execute_online_tensor_decomposition(X_stream, rank, n_iter, ul=5, ll=3, verbose=False, methods=['onlinecp', 'dtd'])<jupyter_output><ipython-input-7-b5523a927d4f>:13: UserWarning: Number of iteration in online cp should not exceed 1 (5 to 1).
  warn('Number of iteration in online cp should not exceed 1 ({} to 1).'.format(n_iter))<jupyter_text>Usage for Online Tensor Decomposition* Create a tensor stream (sum of batch sizes should match with total size of the tensor)  * `create_tensor_stream(tensor, start_to_stream, batch_sizes)`* Invoke online tensor decomposition  * `online_tensor_decomposition(tensor stream, rank, verbose, method)`* Construct an estimated tensor w. factors, leaving weights (=identity matrix)  * `construct_tensor(factors)`<jupyter_code># example
X = sample_video
X_stream = create_tensor_stream(X, start_to_stream=10, batch_sizes=np.full((39), 5, dtype=int))
ktensors, verbose_list = online_tensor_decomposition(X_stream, rank=3, n_iter=1, verbose=False, method='onlinecp')

X_est = construct_tensor(ktensors[0][1])
for (weights, factors) in ktensors[1:]:
    X_est = tl.tensor(np.concatenate((X_est, construct_tensor(factors))))
compare_tensors(X, X_est)
print_tensor(np.asarray((X, X_est))[:,100,100,100:110,0])<jupyter_output>making init decomposition result: 3.057133197784424

 >> onlinecp rank-3 n_iter-1 transformed-False
init_time: 0.05852508544921875
1th_iter: 0.05229616165161133 13029.78632848998 0
2th_iter: 0.05819869041442871 13699.374720316895 -0.535827920203835
3th_iter: 0.05153393745422363 14028.73120149932 -0.3536713909548643
4th_iter: 0.05148768424987793 14980.581378584227 0.060101089549867155
5th_iter: 0.05136513710021973 14998.070978287751 0.0635000762415962
6th_iter: 0.054677486419677734 15057.304195172326 0.08944680258004152
7th_iter: 0.06207609176635742 15014.63620830702 0.06002866033671755
8th_iter: 0.05162858963012695 14888.773405210251 -0.019411196517460264
9th_iter: 0.05153536796569824 14745.870021083269 -0.11084821665990587
10th_iter: 0.0514984130859375 14665.560525238127 -0.16078301545635393
11th_iter: 0.052742958068847656 14349.930683444609 -0.3815967813575657
12th_iter: 0.08582663536071777 14007.849252457181 -0.6215912148049366
13th_iter: 0.04984164237976074 13906.462453946873 -0.66[...]
{'fraction_non_alphanumeric': 0.1409607284176488, 'fraction_numerical': 0.04846189848104497, 'mean_word_length': 2.7219319081551863, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 62, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '636406', 'license': 'MIT', 'n_tokens_mistral': 9281, 'n_tokens_neox': 8483, 'n_words': 1675, 'path': 'Proposed Method/.ipynb_checkpoints/2_drastic_data_handling-checkpoint.ipynb', 'repo_name': 'lucetre/dao-cp', 'size': '23503'}