<commit_before>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os.path

import numpy as np
from numpy.testing import assert_allclose

import abel
from abel.basex import get_bs_cached, cache_cleanup
from abel.tools.analytical import GaussianAnalytical


DATA_DIR = os.path.join(os.path.split(__file__)[0], 'data')


def get_basis_file_name(n, sigma):
    return os.path.join(DATA_DIR, 'basex_basis_{}_{}.npy'.format(n, sigma))


def test_basex_basis_sets_cache():
    # n = 61  (121 full width)
    # sigma = 1  (nbf = 61)
    n = 61
    sigma = 1.0
    file_name = get_basis_file_name(n, sigma)
    if os.path.exists(file_name):
        os.remove(file_name)
    # 1st call generate and save
    get_bs_cached(n, basis_dir=DATA_DIR, verbose=False)
    # 2nd call load from file
    get_bs_cached(n, basis_dir=DATA_DIR, verbose=False)
    if os.path.exists(file_name):
        os.remove(file_name)


def basex_basis_sets_resize(sigma):
    n_s = 50
    n_l = 100
    file_name_s = get_basis_file_name(n_s, sigma)
    file_name_l = get_basis_file_name(n_l, sigma)

    # (remove both basis files)
    def remove_files():
        if os.path.exists(file_name_s):
            os.remove(file_name_s)
        if os.path.exists(file_name_l):
            os.remove(file_name_l)

    # make sure that basis files do not exist and are not cached
    remove_files()
    cache_cleanup()
    # generate small basis and save
    Ai_s = get_bs_cached(n_s, basis_dir=DATA_DIR, verbose=False)
    cache_cleanup()
    # extend to large basis and save
    Ai_s_l = get_bs_cached(n_l, basis_dir=DATA_DIR, verbose=False)
    cache_cleanup()
    # delete basis files
    remove_files()
    # generate large basis and save
    Ai_l = get_bs_cached(n_l, basis_dir=DATA_DIR, verbose=False)
    cache_cleanup()
    # crop large basis to small
    Ai_l_s = get_bs_cached(n_s, basis_dir=DATA_DIR, verbose=False)
    cache_cleanup()
    # delete basis files (clean-up)
    remove_files()

    assert np.array_equal(Ai_s, Ai_l_s)
    assert np.array_equal(Ai_l, Ai_s_l)


def test_basex_basis_sets_resize_1():
    """Test basis resize with default sigma=1"""
    basex_basis_sets_resize(1)


def test_basex_basis_sets_resize_1_5():
    """Test basis resize with sigma=1.5
       (with n not aligned)"""
    basex_basis_sets_resize(1.5)


def test_basex_shape():
    n = 21
    x = np.ones((n, n), dtype='float32')
    Ai = abel.basex.get_bs_cached(n, basis_dir=None, verbose=False)

    recon = abel.basex.basex_core_transform(x, Ai)

    assert recon.shape == (n, n)

<commit_msg>Actually pass sigma to get_bs_cached. Check results with 1e-15 tolerance.<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os.path

import numpy as np
from numpy.testing import assert_allclose

import abel
from abel.basex import get_bs_cached, cache_cleanup
from abel.tools.analytical import GaussianAnalytical


DATA_DIR = os.path.join(os.path.split(__file__)[0], 'data')


def get_basis_file_name(n, sigma):
    return os.path.join(DATA_DIR, 'basex_basis_{}_{}.npy'.format(n, sigma))


def test_basex_basis_sets_cache():
    # n = 61  (121 full width)
    # sigma = 1  (nbf = 61)
    n = 61
    sigma = 1.0
    file_name = get_basis_file_name(n, sigma)
    if os.path.exists(file_name):
        os.remove(file_name)
    # 1st call generate and save
    get_bs_cached(n, basis_dir=DATA_DIR, verbose=False)
    # 2nd call load from file
    get_bs_cached(n, basis_dir=DATA_DIR, verbose=False)
    if os.path.exists(file_name):
        os.remove(file_name)


def basex_basis_sets_resize(sigma):
    n_s = 50
    n_l = 100
    file_name_s = get_basis_file_name(n_s, sigma)
    file_name_l = get_basis_file_name(n_l, sigma)

    # (remove both basis files)
    def remove_files():
        if os.path.exists(file_name_s):
            os.remove(file_name_s)
        if os.path.exists(file_name_l):
            os.remove(file_name_l)

    # make sure that basis files do not exist and are not cached
    remove_files()
    cache_cleanup()
    # generate small basis and save
    Ai_s = get_bs_cached(n_s, sigma, basis_dir=DATA_DIR, verbose=False)
    cache_cleanup()
    # extend to large basis and save
    Ai_s_l = get_bs_cached(n_l, sigma, basis_dir=DATA_DIR, verbose=False)
    cache_cleanup()
    # delete basis files
    remove_files()
    # generate large basis and save
    Ai_l = get_bs_cached(n_l, sigma, basis_dir=DATA_DIR, verbose=False)
    cache_cleanup()
    # crop large basis to small
    Ai_l_s = get_bs_cached(n_s, sigma, basis_dir=DATA_DIR, verbose=False)
    cache_cleanup()
    # delete basis files (clean-up)
    remove_files()

    assert_allclose(Ai_s, Ai_l_s, atol=1e-15, rtol=1e-15)
    assert_allclose(Ai_l, Ai_s_l, atol=1e-15, rtol=1e-15)


def test_basex_basis_sets_resize_1():
    """Test basis resize with default sigma=1"""
    basex_basis_sets_resize(1)


def test_basex_basis_sets_resize_1_5():
    """Test basis resize with sigma=1.5
       (with n not aligned)"""
    basex_basis_sets_resize(1.5)


def test_basex_shape():
    n = 21
    x = np.ones((n, n), dtype='float32')
    Ai = abel.basex.get_bs_cached(n, basis_dir=None, verbose=False)

    recon = abel.basex.basex_core_transform(x, Ai)

    assert recon.shape == (n, n)


{'fraction_non_alphanumeric': 0.09757462686567164, 'fraction_numerical': 0.015111940298507463, 'mean_word_length': 3.6496097137901127, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4003926', 'n_tokens_mistral': 2084, 'n_tokens_neox': 1940, 'n_words': 449}