<commit_before>"""f90nml.py
Parse fortran namelist files into dicts of standard Python data types.
Contact: <NAME> <<EMAIL>>
---
Distributed as part of f90nml, Copyright 2014 <NAME>
Licensed under the Apache License, Version 2.0
http://www.apache.org/licenses/LICENSE-2.0
"""

from collections import OrderedDict
import itertools
import os
import shlex

__version__ = '0.4'


#---
def read(nml_fname, verbose=False):
    """Parse a Fortran 90 namelist file and store the contents in a ``dict``.

    >>> data_nml = f90nml.read('data.nml')"""

    nml_file = open(nml_fname, 'r')

    f90lex = shlex.shlex(nml_file)
    f90lex.commenters = '!'
    f90lex.escapedquotes = '\'"'
    f90lex.wordchars += '.-+'      # Include floating point characters
    tokens = iter(f90lex)

    # Store groups in case-insensitive dictionary
    nmls = NmlDict()

    for t in tokens:

        # Ignore tokens outside of namelist groups
        while t != '&':
            t, prior_t = next(tokens), t

        # Current token is now '&'

        # Create the next namelist
        g_name = next(tokens)
        g_vars = NmlDict()

        v_name = None
        v_idx = None
        v_vals = []

        # Current token is either a variable name or finalizer (/, &)

        # Populate the namelist group
        while g_name:

            if not v_name:
                t, prior_t = next(tokens), t

                # Skip commas separating objects
                if t == ',':
                    t, prior_t = next(tokens), t

            # Diagnostic testing
            if verbose:
                print('  tokens: {} {}'.format(prior_t, t))

            while v_name:

                t, prior_t = next(tokens), t

                if verbose:
                    print('    vstate: {} {} {}'.format(v_name, v_idx, v_vals))
                    print('    tokens: {} {}'.format(prior_t, t))

                # Parse the prior token value
                # TODO: Add '%' to first tuple
                if (not t in ('(', '=') or prior_t == '=') \
                        and not (prior_t, t) == (',', '/'):
                    # Parse the variable string
                    if prior_t in ('=', ','):
                        if t in (',', '/', '&'):
                            next_value = None
                        else:
                            continue
                    else:
                        next_value, t = parse_f90val(tokens, t, prior_t)

                    if v_idx:

                        v_i = next(v_idx)

                        if v_name in g_vars:
                            v_vals = g_vars[v_name]
                            if type(v_vals) != list:
                                v_vals = [v_vals]

                        try:
                            # Default Fortran indexing starts at 1
                            v_vals[v_i - 1] = next_value
                        except IndexError:
                            # Expand list to accomodate out-of-range indices
                            size = len(v_vals)
                            v_vals.extend(None for i in range(size, v_i))
                            v_vals[v_i - 1] = next_value
                    else:
                        v_vals.append(next_value)

                # Save then deactivate the current variable
                # TODO: Add '%'
                if t in ('(', '=', '/', '&'):

                    if len(v_vals) == 0:
                        v_vals = None
                    elif len(v_vals) == 1:
                        v_vals = v_vals[0]

                    g_vars[v_name] = v_vals

                    v_name = None
                    v_vals = []

            # Set the next active variable
            # TODO: Add '%' to the list
            if t in ('=', '('):
                v_name = prior_t
                v_idx = None

                # Parse the indices of the current variable
                if t == '(':
                    v_name, v_indices, t = parse_f90idx(tokens, t, prior_t)

                    # TODO: Multidimensional support
                    # TODO: End index support (currently ignored)
                    i_s = 1 if not v_indices[0][0] else v_indices[0][0]
                    i_r = 1 if not v_indices[0][2] else v_indices[0][2]
                    v_idx = itertools.count(i_s, i_r)

                # Identify any derived type fields
                if t == '%':
                    # TODO: Recurse into a new parsing of variable name
                    pass

            # Finalise namelist group
            if t in ('/', '&'):
                # Test for classic namelist finaliser
                if t == '&':
                    t, prior_t = next(tokens), t
                    assert t.lower() == 'end'

                # Append the grouplist to the namelist (including empty groups)
                nmls[g_name] = g_vars
                g_name, g_vars = None, None

    nml_file.close()

    return nmls


#---
def write(nml, nml_fname, force=False):
    """Output dict to a Fortran 90 namelist file."""

    if not force and os.path.isfile(nml_fname):
        raise IOError('File {} already exists.'.format(nml_fname))

    nml_file = open(nml_fname, 'w')

    for grp in nml.keys():
        nml_file.write('&{}\n'.format(grp))

        grp_vars = nml[grp]
        for v_name in grp_vars.keys():

            v_val = grp_vars[v_name]

            if type(v_val) == list:
                v_str = ', '.join([to_f90str(v) for v in v_val])
            else:
                v_str = to_f90str(v_val)

            nml_file.write('    {} = {}\n'.format(v_name, v_str))

        nml_file.write('/\n')

    nml_file.close()


#---
def to_f90str(value):
    """Convert primitive Python types to equivalent Fortran strings"""

    if type(value) is int:
        return str(value)
    elif type(value) is float:
        return str(value)
    elif type(value) is bool:
        return '.{}.'.format(str(value).lower())
    elif type(value) is complex:
        return '({}, {})'.format(value.real, value.imag)
    elif type(value) is str:
        return '\'{}\''.format(value)
    elif value is None:
        return ''
    else:
        raise ValueError('Type {} of {} cannot be converted to a Fortran type.'<commit_msg>Separate varname parsing fn; dummy value parsing fn<commit_after>"""f90nml.py
Parse fortran namelist files into dicts of standard Python data types.
Contact: <NAME> <<EMAIL>>
---
Distributed as part of f90nml, Copyright 2014 <NAME>
Licensed under the Apache License, Version 2.0
http://www.apache.org/licenses/LICENSE-2.0
"""

from collections import OrderedDict
import itertools
import os
import shlex

__version__ = '0.4'


#---
def read(nml_fname, verbose=False):
    """Parse a Fortran 90 namelist file and store the contents in a ``dict``.

    >>> data_nml = f90nml.read('data.nml')"""

    nml_file = open(nml_fname, 'r')

    f90lex = shlex.shlex(nml_file)
    f90lex.commenters = '!'
    f90lex.escapedquotes = '\'"'
    f90lex.wordchars += '.-+'      # Include floating point characters
    tokens = iter(f90lex)

    # Store groups in case-insensitive dictionary
    nmls = NmlDict()

    for t in tokens:

        # Ignore tokens outside of namelist groups
        while t != '&':
            t, prior_t = next(tokens), t

        # Current token is now '&'

        # Create the next namelist
        g_name = next(tokens)
        g_vars = NmlDict()

        v_name = None
        v_idx = None
        v_vals = []

        # Current token is either a variable name or finalizer (/, &)

        # Populate the namelist group
        while g_name:

            if not v_name:
                t, prior_t = next(tokens), t

                # Skip commas separating objects
                if t == ',':
                    t, prior_t = next(tokens), t

            # Diagnostic testing
            if verbose:
                print('  tokens: {} {}'.format(prior_t, t))

            while v_name:

                t, prior_t = next(tokens), t

                if verbose:
                    print('    vstate: {} {} {}'.format(v_name, v_idx, v_vals))
                    print('    tokens: {} {}'.format(prior_t, t))

                # Parse the prior token value
                # TODO: Add '%' to first tuple
                if (not t in ('(', '=') or prior_t == '=') \
                        and not (prior_t, t) == (',', '/'):
                    # Parse the variable string
                    if prior_t in ('=', ','):
                        if t in (',', '/', '&'):
                            next_value = None
                        else:
                            continue
                    else:
                        next_value, t = parse_f90val(tokens, t, prior_t)

                    if v_idx:

                        v_i = next(v_idx)

                        if v_name in g_vars:
                            v_vals = g_vars[v_name]
                            if type(v_vals) != list:
                                v_vals = [v_vals]

                        try:
                            # Default Fortran indexing starts at 1
                            v_vals[v_i - 1] = next_value
                        except IndexError:
                            # Expand list to accomodate out-of-range indices
                            size = len(v_vals)
                            v_vals.extend(None for i in range(size, v_i))
                            v_vals[v_i - 1] = next_value
                    else:
                        v_vals.append(next_value)

                # Save then deactivate the current variable
                # TODO: Add '%'
                if t in ('(', '=', '/', '&'):

                    if len(v_vals) == 0:
                        v_vals = None
                    elif len(v_vals) == 1:
                        v_vals = v_vals[0]

                    g_vars[v_name] = v_vals

                    v_name = None
                    v_vals = []

            # Set the next active variable
            # TODO: Add '%' to the list
            if t in ('=', '(', '%'):
                v_name, v_idx, v_value, t = parse_vname(tokens, t, prior_t)

            # Finalise namelist group
            if t in ('/', '&'):
                # Test for classic namelist finaliser
                if t == '&':
                    t, prior_t = next(tokens), t
                    assert t.lower() == 'end'

                # Append the grouplist to the namelist (including empty groups)
                nmls[g_name] = g_vars
                g_name, g_vars = None, None

    nml_file.close()

    return nmls


#---
def write(nml, nml_fname, force=False):
    """Output dict to a Fortran 90 namelist file."""

    if not force and os.path.isfile(nml_fname):
        raise IOError('File {} already exists.'.format(nml_fname))

    nml_file = open(nml_fname, 'w')

    for grp in nml.keys():
        nml_file.write('&{}\n'.format(grp))

        grp_vars = nml[grp]
        for v_name in grp_vars.keys():

            v_val = grp_vars[v_name]

            if type(v_val) == list:
                v_str = ', '.join([to_f90str(v) for v in v_val])
            else:
                v_str = to_f90str(v_val)

            nml_file.write('    {} = {}\n'.format(v_name, v_str))

        nml_file.write('/\n')

    nml_file.close()


#---
def parse_vname(tokens, t, prior_t):

    v_name = prior_t

    # Parse the indices of the current variable
    if t == '(':
        v_name, v_indices, t = parse_f90idx(tokens, t, prior_t)

        # TODO: Multidimensional support
        # TODO: End index support (currently ignored)
        i_s = 1 if not v_indices[0][0] else v_indices[0][0]
        i_r = 1 if not v_indices[0][2] else v_indices[0][2]
        v_idx = itertools.count(i_s, i_r)
    else:
        v_idx = None

    # Identify any derived type fields
    if t == '%':
        v_value = parse_vname(tokens, t, prior_t)
    else:
        v_value = parse_vvalue(tokens, t, prior_t)

    return v_name, v_idx, v_value, t


#---
def parse_vvalue(tokens, t, prior_t):
    # TODO: write me (^_^)
    pass


#---
def to_f90str(value):
    """Convert primitive Python types to equivalent Fortran strings"""

    if type(value) is int:
        return str(value)
    elif type(value) is float:
        return str(value)
    elif type(value) is bool:
        return '.{}.'.format(str(value).lower())
    elif type(value) is complex:
        return '({}, {})'.format(value.real, value.imag)
    elif type(value) is str:
        return '\'{}\''.format(value)
    elif value is None:
        return ''
    else:
        raise ValueError('Type {} of {} cannot be converted to a Fortran type.'
{'fraction_non_alphanumeric': 0.11099724083563264, 'fraction_numerical': 0.008829325975561688, 'mean_word_length': 1.3886273771417812, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4453781', 'n_tokens_mistral': 3986, 'n_tokens_neox': 3682, 'n_words': 1048}