<commit_before>import os
from ... import lut


def read_turbomole(basis_path):
    '''Reads a turbomole-formatted file and converts it to a dictionary with the
       usual BSE fields

       Note that the turbomole format does not store all the fields we
       have, so some fields are left blank
    '''

    skipchars = '*#'

    if not os.path.isfile(basis_path):
        raise RuntimeError('Turbomole basis set path \'{}\' does not exist'.format(basis_path))

    fname = os.path.basename(basis_path)

    with open(basis_path, 'r') as f:
        basis_lines = [l.strip() for l in f]
        basis_lines = [l for l in basis_lines if l and not l[0] in skipchars]

    bs_data = {
        'molssi_bse_schema': {
            'schema_type': 'component',
            'schema_version': '0.1'
        },
        'basis_set_description': fname,
        'basis_set_references': [],
        'basis_set_elements': {}
    }

    i = 0
    while i < len(basis_lines):
        line = basis_lines[i]
        elementsym = line.split()[0]

        element_Z = lut.element_Z_from_sym(elementsym)
        element_Z = str(element_Z)

        if not element_Z in bs_data['basis_set_elements']:
            bs_data['basis_set_elements'][element_Z] = {}

        eldata = bs_data['basis_set_elements'][element_Z]

        if "ecp" in line.lower():
            if not 'element_ecp' in eldata:
                eldata['element_ecp'] = []

            i += 1
            line = basis_lines[i]

            lsplt = line.split()
            maxam = int(lsplt[5])
            n_elec = int(lsplt[2])

            amlist = [maxam]
            amlist.extend(list(range(0, maxam)))

            i += 1
            for shell_am in amlist:
                shell_am2 = lut.amchar_to_int(basis_lines[i][0])[0]
                if shell_am2 != shell_am:
                    raise RuntimeError("AM not in expected order?")

                i += 1<commit_msg>Harden tmol reader a bit<commit_after>import os
from ... import lut


def read_turbomole(basis_path):
    '''Reads a turbomole-formatted file and converts it to a dictionary with the
       usual BSE fields

       Note that the turbomole format does not store all the fields we
       have, so some fields are left blank
    '''

    skipchars = '*#$'

    if not os.path.isfile(basis_path):
        raise RuntimeError('Turbomole basis set path \'{}\' does not exist'.format(basis_path))

    fname = os.path.basename(basis_path)

    with open(basis_path, 'r') as f:
        basis_lines = [l.strip() for l in f]
        basis_lines = [l for l in basis_lines if l and not l[0] in skipchars]

    bs_data = {
        'molssi_bse_schema': {
            'schema_type': 'component',
            'schema_version': '0.1'
        },
        'basis_set_description': fname,
        'basis_set_references': [],
        'basis_set_elements': {}
    }

    i = 0
    while i < len(basis_lines):
        line = basis_lines[i]
        elementsym = line.split()[0]

        element_Z = lut.element_Z_from_sym(elementsym)
        element_Z = str(element_Z)

        if not element_Z in bs_data['basis_set_elements']:
            bs_data['basis_set_elements'][element_Z] = {}

        eldata = bs_data['basis_set_elements'][element_Z]

        if "ecp" in line.lower():
            if not 'element_ecp' in eldata:
                eldata['element_ecp'] = []

            i += 1
            line = basis_lines[i]

            lsplt = line.split('=')
            maxam = int(lsplt[2])
            n_elec = int(lsplt[1].split()[0])

            amlist = [maxam]
            amlist.extend(list(range(0, maxam)))

            i += 1
            for shell_am in amlist:
                shell_am2 = lut.amchar_to_int(basis_lines[i][0])[0]
                if shell_am2 != shell_am:
                    raise RuntimeError("AM not in expected order?")

                i += 1
{'fraction_non_alphanumeric': 0.11027892561983471, 'fraction_numerical': 0.008006198347107437, 'mean_word_length': 1.9838212634822805, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1206970', 'n_tokens_mistral': 1310, 'n_tokens_neox': 1220, 'n_words': 309}