<commit_before>"""f90nml.py
Parse fortran namelist files into dicts of standard Python data types.
Contact: <NAME> <<EMAIL>>
---
Distributed as part of f90nml, Copyright 2014 <NAME>
Licensed under the Apache License, Version 2.0
http://www.apache.org/licenses/LICENSE-2.0
"""
import os
import re
import shlex

__version__ = '0.2dev'

#---
def read(nml_fname):

    f = open(nml_fname, 'r')

    f90 = shlex.shlex(f)
    f90.commenters = '!'
    f90.escapedquotes = '\'"'
    f90.wordchars += '.-+'      # Include floating point characters
    tokens = iter(f90)

    # Store groups in case-insensitive dictionary
    nmls = NmlDict()

    for t in tokens:

        # Ignore tokens outside of namelist groups
        while t != '&':
            t = next(tokens)

        # Current token is now '&'

        g_name = next(tokens)
        g_vars = NmlDict()

        v_name = None
        v_idx = None
        v_vals = []

        prior_t = t
        t = next(tokens)

        # Current token is either a variable name or finalizer (/)

        while t != '/':

            # Skip commas
            if t == ',':
                prior_t = t
                t = next(tokens)

            # Advance token
            if not t == '/':
                prior_t = t
                t = next(tokens)

            if v_name:

                #---
                # Parse the prior token value

                # Skip variable names and end commas
                if not t in ('(', '=') and not (prior_t, t) == (',', '/'):

                    # Skip ahead on first value
                    if prior_t == '=':
                        prior_t = t
                        t = next(tokens)

                    # Parse the variable string
                    if prior_t == ',' :
                        next_value = None
                    else:
                        next_value = from_f90str(prior_t)

                    if v_idx:

                        v_i = next(v_idx)

                        if v_name in g_vars:
                            v_vals = g_vars[v_name]
                            if type(v_vals) != list:
                                v_vals = [v_vals]

                        try:
                            # Default Fortran indexing starts at 1
                            v_vals[v_i - 1] = next_value
                        except IndexError:
                            # Expand list to accomodate out-of-range indices
                            size = len(v_vals)
                            v_vals.extend(None for i in range(size, v_i))
                            v_vals[v_i - 1] = next_value
                    else:
                        v_vals.append(next_value)

                # Save then deactivate the current variable
                if t in ('(', '=', '/'):

                    if len(v_vals) == 1:
                        v_vals = v_vals[0]
                    g_vars[v_name] = v_vals

                    v_name = None
                    v_vals = []

            # Parse the indices of the current variable
            if t == '(':
                v_name, v_indices, t = parse_f90idx(tokens, t, prior_t)
                v_idx = gen_index(v_indices)

            if t == '=':
                # Activate the next variable
                if not v_name:
                    v_name = prior_t
                    v_idx = None

        # Append the grouplist to the namelist (including empty groups)
        nmls[g_name] = g_vars

    f.close()

    return nmls


#---
def write(nml, nml_fname):

    if os.path.isfile(nml_fname):
        raise IOError('File {} already exists.'.format(nml_fname))

    f = open(nml_fname, 'w')

    for grp in sorted(nml.keys()):
        f.write('&{}\n'.format(grp))

        grp_vars = nml[grp]
        for v_name in sorted(grp_vars.keys()):

            v_val = grp_vars[v_name]

            if type(v_val) == list:
                v_str = ', '.join([to_f90str(v) for v in v_val])
            else:
                v_str = to_f90str(v_val)

            f.write('    {} = {}\n'.format(v_name, v_str))

        f.write('/\n')

    f.close()


#---
def to_f90str(v):
    """Convert primitive Python types to equivalent Fortran strings"""

    # TODO: Hash this somehow
    if type(v) is int:
        return str(v)
    elif type(v) is float:
        # TODO: Floating point precision?
        return str(v)
    elif type(v) is bool:
        return '.{}.'.format(str(v).lower())
    elif type(v) is complex:
        return '({}, {})'.format(v.real, v.imag)
    elif type(v) is str:
        return '\'{}\''.format(v)
    elif v is None:
        return ''
    else:
        raise ValueError('Type {} of {} cannot be converted to a Fortran type.'
                         ''.format(type(v), v))


#---
def from_f90str(s):
    """Convert string repr of Fortran type to equivalent Python type."""
    assert type(s) is str
<commit_msg>complex does not break, but is also not parsed<commit_after>"""f90nml.py
Parse fortran namelist files into dicts of standard Python data types.
Contact: <NAME> <<EMAIL>>
---
Distributed as part of f90nml, Copyright 2014 <NAME>
Licensed under the Apache License, Version 2.0
http://www.apache.org/licenses/LICENSE-2.0
"""
import os
import re
import shlex

__version__ = '0.2dev'

#---
def read(nml_fname):

    f = open(nml_fname, 'r')

    f90 = shlex.shlex(f)
    f90.commenters = '!'
    f90.escapedquotes = '\'"'
    f90.wordchars += '.-+'      # Include floating point characters
    tokens = iter(f90)

    # Store groups in case-insensitive dictionary
    nmls = NmlDict()

    for t in tokens:

        # Ignore tokens outside of namelist groups
        while t != '&':
            t = next(tokens)

        # Current token is now '&'

        g_name = next(tokens)
        g_vars = NmlDict()

        v_name = None
        v_idx = None
        v_vals = []

        prior_t = t
        t = next(tokens)

        # Current token is either a variable name or finalizer (/)

        while t != '/':

            # Skip commas
            if t == ',':
                prior_t = t
                t = next(tokens)

            # Advance token
            if not t == '/':
                prior_t = t
                t = next(tokens)

            print('tokens: {} {}'.format(prior_t, t))

            if v_name:

                #---
                # Parse the prior token value

                # Skip variable names and end commas
                if not t in ('(', '=') and not (prior_t, t) == (',', '/'):

                    # Skip ahead on first value
                    if prior_t == '=':
                        prior_t = t
                        t = next(tokens)

                    # Parse the variable string
                    if prior_t == ',' :
                        next_value = None
                    else:
                        next_value = parse_f90val(tokens, t, prior_t)

                    if v_idx:

                        v_i = next(v_idx)

                        if v_name in g_vars:
                            v_vals = g_vars[v_name]
                            if type(v_vals) != list:
                                v_vals = [v_vals]

                        try:
                            # Default Fortran indexing starts at 1
                            v_vals[v_i - 1] = next_value
                        except IndexError:
                            # Expand list to accomodate out-of-range indices
                            size = len(v_vals)
                            v_vals.extend(None for i in range(size, v_i))
                            v_vals[v_i - 1] = next_value
                    else:
                        v_vals.append(next_value)

                # Save then deactivate the current variable
                if t in ('(', '=', '/'):

                    if len(v_vals) == 1:
                        v_vals = v_vals[0]
                    g_vars[v_name] = v_vals

                    v_name = None
                    v_vals = []

            # Parse the indices of the current variable
            if t == '(' and not prior_t == '=':
                v_name, v_indices, t = parse_f90idx(tokens, t, prior_t)
                v_idx = gen_index(v_indices)

            if t == '=':
                # Activate the next variable
                if not v_name:
                    v_name = prior_t
                    v_idx = None

        # Append the grouplist to the namelist (including empty groups)
        nmls[g_name] = g_vars

    f.close()

    return nmls


#---
def write(nml, nml_fname):

    if os.path.isfile(nml_fname):
        raise IOError('File {} already exists.'.format(nml_fname))

    f = open(nml_fname, 'w')

    for grp in sorted(nml.keys()):
        f.write('&{}\n'.format(grp))

        grp_vars = nml[grp]
        for v_name in sorted(grp_vars.keys()):

            v_val = grp_vars[v_name]

            if type(v_val) == list:
                v_str = ', '.join([to_f90str(v) for v in v_val])
            else:
                v_str = to_f90str(v_val)

            f.write('    {} = {}\n'.format(v_name, v_str))

        f.write('/\n')

    f.close()


#---
def to_f90str(v):
    """Convert primitive Python types to equivalent Fortran strings"""

    # TODO: Hash this somehow
    if type(v) is int:
        return str(v)
    elif type(v) is float:
        # TODO: Floating point precision?
        return str(v)
    elif type(v) is bool:
        return '.{}.'.format(str(v).lower())
    elif type(v) is complex:
        return '({}, {})'.format(v.real, v.imag)
    elif type(v) is str:
        return '\'{}\''.format(v)
    elif v is None:
        return ''
    else:
        raise ValueError('Type {} of {} cannot be converted to a Fortran type.'
                         ''.format(type(v), v))


#---
def parse_f90val(tokens, t, s):
    """Convert string repr of Fortran type to equivalent Python type."""
    assert type(s) is str

{'fraction_non_alphanumeric': 0.10869346227460461, 'fraction_numerical': 0.0082603001913972, 'mean_word_length': 1.2770642201834863, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5117976', 'n_tokens_mistral': 3110, 'n_tokens_neox': 2884, 'n_words': 816}