<commit_before>"""
Tests BSE curation functions
"""

import os
import tempfile

from basis_set_exchange import curate, api, manip, validator
from .common_testvars import auth_data_dir

# yapf: disable
# Mapping of filename to refs
_source_data = {'cc-pvdz.1.gbasis.bz2': None,
                'def2-tzvp.1.tm.bz2': 'myref',
                'aug-pcj-1.1.gbs.bz2': ['ref1', 'ref2'],
                'def2-ecp.1.tm.bz2': {'rb': 'ref1', 47: 'ref2', '48': 'ref3', '49-xe': 'ref4'}
}
# yapf: enable


def test_add_basis(tmp_path):
    tmp_path = str(tmp_path)  # Needed for python 3.5
    for sf, refs in _source_data.items():
        sf_path = os.path.join(auth_data_dir, sf)

        name = sf.split('.')[0]
        curate.add_basis(sf_path, tmp_path, 'test_subdir', 'test_' + name, 'test_basis_' + name, 'test_family',
                         'orbital', 'Test Basis Description: ' + name, '1', 'Test Basis Revision Description', refs)

    md = api.get_metadata(tmp_path)
    assert len(md) == len(_source_data)

    # Re-read
    for sf in _source_data.keys():
        name = name = sf.split('.')[0]
        name = 'test_basis_' + name
        bse_dict = api.get_basis(name, data_dir=tmp_path)

        assert bse_dict['basis_set_family'] == 'test_family'

        # Compare against the file we created from
        sf_path = os.path.join(auth_data_dir, sf)
        test_dict = curate.read_formatted_basis(sf_path)

        # Compare, ignoring metadata (not stored in most formats)
        assert curate.compare_basis(bse_dict, test_dict, rel_tol=0.0)

    # Validate the new data dir
    validator.validate_data_dir(tmp_path)<commit_msg>Improve testing of add_basis<commit_after>"""
Tests BSE curation functions
"""

import os
import tempfile

from basis_set_exchange import curate, api, validator, misc, fileio, compose, memo
from .common_testvars import auth_data_dir

# yapf: disable
# Mapping of filename to refs
_source_data = {'cc-pvdz.1.gbasis.bz2': None,
                'def2-tzvp.1.tm.bz2': 'myref',
                'aug-pcj-1.1.gbs.bz2': ['ref1', 'ref2'],
                'def2-ecp.1.tm.bz2': {'rb': 'ref1', 47: 'ref2', '48': 'ref3', '49-xe': 'ref4'}
}

_expected_files = {'test_subdir/test_aug-pcj-1_ref1_ref2.1.json': 'H-Ar',
                   'test_subdir/test_cc-pvdz_noref.1.json':       'H-Ar,Ca-Kr',
                   'test_subdir/test_def2-ecp_noref.1.json':      'Sr-Pd,Cs-Rn',
                   'test_subdir/test_def2-ecp_ref1.1.json':       'Rb',
                   'test_subdir/test_def2-ecp_ref2.1.json':       'Ag',
                   'test_subdir/test_def2-ecp_ref3.1.json':       'Cd',
                   'test_subdir/test_def2-ecp_ref4.1.json':       'In-Xe',
                   'test_subdir/test_def2-tzvp_myref.1.json':     'H-Rn',
                   'test_subdir/test_def2-tzvp.1.element.json':   'H-Rn',
                   'test_subdir/test_aug-pcj-1.1.element.json':   'H-Ar',
                   'test_subdir/test_def2-ecp.1.element.json':    'Rb-Rn',
                   'test_subdir/test_cc-pvdz.1.element.json':     'H-Ar,Ca-Kr',
                   'test_aug-pcj-1.1.table.json':                 'H-Ar',
                   'test_cc-pvdz.1.table.json':                   'H-Ar,Ca-Kr',
                   'test_def2-ecp.1.table.json':                  'Rb-Rn',
                   'test_def2-tzvp.1.table.json':                 'H-Rn'
}
# yapf: enable


def test_add_basis(tmp_path):
    tmp_path = str(tmp_path)  # Needed for python 3.5
    for sf, refs in _source_data.items():
        sf_path = os.path.join(auth_data_dir, sf)

        name = sf.split('.')[0]
        curate.add_basis(sf_path, tmp_path, 'test_subdir', 'test_' + name, 'test_basis_' + name, 'test_family',
                         'orbital', 'Test Basis Description: ' + name, '1', 'Test Basis Revision Description', refs)

    md = api.get_metadata(tmp_path)
    assert len(md) == len(_source_data)

    # Re-read
    for sf in _source_data.keys():
        name = name = sf.split('.')[0]
        name = 'test_basis_' + name
        bse_dict = api.get_basis(name, data_dir=tmp_path)

        assert bse_dict['basis_set_family'] == 'test_family'

        # Compare against the file we created from
        sf_path = os.path.join(auth_data_dir, sf)
        assert curate.compare_basis_against_ref(name, sf_path, data_dir=tmp_path) == True

        # Check that all the files exist and contain the right elements
        for fpath, elements in _expected_files.items():
            fpath = os.path.join(tmp_path, fpath)
            assert os.path.isfile(fpath)

            fdata = fileio.read_json_basis(fpath)
            expect_elements = set(misc.expand_elements(elements, True))
            assert set(fdata['basis_set_elements'].keys()) == expect_elements

    # Validate the new data dir
    validator.validate_data_dir(tmp_path)
{'fraction_non_alphanumeric': 0.1360628618693135, 'fraction_numerical': 0.019230769230769232, 'mean_word_length': 2.331267217630854, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4998381', 'n_tokens_mistral': 1825, 'n_tokens_neox': 1724, 'n_words': 328}