<commit_before>import numpy as np
from .cpdirect import _cp_initialize
from .solvers import ls_solver
from time import time
from .kruskal import standardize_factors

# default options for randomized solver
OPTIONS = {
    'min_time': 0,
    'max_time': np.inf,
    'n_iter_max': 10000,
    'print_every': .2,
    'prepend_print': '\r',
    'append_print': '',
    'rs': None,
    'fs': 2**14,
    'rs_inc': 2,
    'rs_max': None,
    'patience': 40,
    'tol': 1e-3,
    'polyak_averages': -1
}
            
def cp_rand(tensor, rank, M=None, l1=None, l2=None, nonneg=False, init=None,
            options=OPTIONS):
    
    # ensure default options are present
    for k, v in OPTIONS.items():
        options.setdefault(k, v)

    # extract options
    rs = options['rs']
    fs = options['fs']
    rs_inc = options['rs_inc']
    rs_max = options['rs_max']
    patience = options['patience']
    tol = options['tol']

    # heuristic for the number of samples
    if rs is None:
        rs = np.maximum(10, int(np.ceil(4 * rank * np.log(rank))))
    
    # search over an order of magnitude for fitting samples
    if rs_max is None:
        rs_max = rs * 10

    # default initialization method
    if init is None:
        init = 'rand' if nonneg is False else 'randn'

    # intialize factor matrices
    factors = _cp_initialize(tensor, rank, init)
    
    # set up error estimation
    fit_ind = np.random.randint(0, tensor.size, size=fs)
    fit_sub = np.array(np.unravel_index(fit_ind, tensor.shape)).T
    tensor_sample = tensor.ravel()[fit_ind]<commit_msg>fix loss estimate for missing data<commit_after>import numpy as np
from .cpdirect import _cp_initialize
from .solvers import ls_solver
from time import time
from .kruskal import standardize_factors

# default options for randomized solver
OPTIONS = {
    'min_time': 0,
    'max_time': np.inf,
    'n_iter_max': 10000,
    'print_every': .2,
    'prepend_print': '\r',
    'append_print': '',
    'rs': None,
    'fs': 2**14,
    'rs_inc': 2,
    'rs_max': None,
    'patience': 40,
    'tol': 1e-3,
    'polyak_averages': -1
}
            
def cp_rand(tensor, rank, M=None, l1=None, l2=None, nonneg=False, init=None,
            options=OPTIONS):
    
    # ensure default options are present
    for k, v in OPTIONS.items():
        options.setdefault(k, v)

    # extract options
    rs = options['rs']
    fs = options['fs']
    rs_inc = options['rs_inc']
    rs_max = options['rs_max']
    patience = options['patience']
    tol = options['tol']

    # heuristic for the number of samples
    if rs is None:
        rs = np.maximum(10, int(np.ceil(4 * rank * np.log(rank))))
    
    # search over an order of magnitude for fitting samples
    if rs_max is None:
        rs_max = rs * 10

    # default initialization method
    if init is None:
        init = 'rand' if nonneg is False else 'randn'

    # intialize factor matrices
    factors = _cp_initialize(tensor, rank, init)
    
    # set up error estimation
    if M is None:
        fit_ind = np.random.randint(0, tensor.size, size=fs)
    else:
        fit_ind = np.random.choice(np.where(tensor.ravel())[0], size=fs, replace=False)
    fit_sub = np.array(np.unravel_index(fit_ind, tensor.shape)).T
    tensor_sample = tensor.ravel()[fit_ind]
{'fraction_non_alphanumeric': 0.12122137404580152, 'fraction_numerical': 0.014961832061068702, 'mean_word_length': 2.7829099307159355, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3586858', 'n_tokens_mistral': 1150, 'n_tokens_neox': 1090, 'n_words': 317}