<commit_before>"""f90nml.parser
   =============

   Fortran namelist parser and tokenizer to convert contents into a hierarchy
   of dicts containing intrinsic Python data types.

   :copyright: Copyright 2014 <NAME>, see AUTHORS for details.
   :license: Apache License, Version 2.0, see LICENSE for details.
"""
import copy
import shlex
from string import whitespace

from f90nml.fpy import pyfloat, pycomplex, pybool, pystr
from f90nml.namelist import Namelist
from f90nml.findex import FIndex


class Parser(object):
    """shlex-based Fortran namelist parser."""

    def __init__(self):

        # Token management
        self.tokens = None
        self.token = None
        self.prior_token = None

        # Patching
        self.pfile = None

        # Control flags
        self._row_major = False
        self._strict_logical = True

        # Configuration
        self.comment_tokens = '!'

    @property
    def row_major(self):
        """Return true if multidimensional arrays are in row-major format."""
        return self._row_major

    @row_major.setter
    def row_major(self, value):
        """Validate and set row-major format for multidimensional arrays."""

        if value is not None:
            if not isinstance(value, bool):
                raise ValueError(
                    'f90nml: error: row_major must be a logical value.')
            else:
                self._row_major = value

    @property
    def strict_logical(self):
        """Return true for strict logical value parsing."""
        return self._strict_logical

    @strict_logical.setter
    def strict_logical(self, value):
        """Validate and set the strict logical flag."""

        if value is not None:
            if not isinstance(value, bool):
                raise ValueError(
                    'f90nml: error: strict_logical must be a logical value.')
            else:
                self._strict_logical = value

    def read(self, nml_fname, nml_patch_in=None, patch_fname=None):
        """Parse a Fortran 90 namelist file and store the contents.

        >>> from f90nml.parser import Parser
        >>> parser = Parser()
        >>> data_nml = parser.read('data.nml')"""

        # Convert patch data to a Namelist object
        if nml_patch_in:
            if not isinstance(nml_patch_in, dict):
                raise ValueError('Input patch must be a dict or a Namelist.')

            nml_patch = copy.deepcopy(Namelist(nml_patch_in))

            if not patch_fname:
                patch_fname = nml_fname + '~'
            elif nml_fname == patch_fname:
                raise ValueError('f90nml: error: Patch filepath cannot be the '
                                 'same as the original filepath.')
            self.pfile = open(patch_fname, 'w')
        else:
            nml_patch = Namelist()

        nml_file = open(nml_fname, 'r')

        f90lex = shlex.shlex(nml_file)
        f90lex.whitespace = ''
        f90lex.wordchars += '.-+'       # Include floating point tokens
        if nml_patch:
            f90lex.commenters = ''
        else:
            f90lex.commenters = self.comment_tokens

        self.tokens = iter(f90lex)

        nmls = Namelist()

        # TODO: Replace "while True" with an update_token() iterator
        self.update_tokens(write_token=False)
        while True:
            try:
                # Check for classic group terminator
                if self.token == 'end':
                    self.update_tokens()

                # Ignore tokens outside of namelist groups
                while self.token not in ('&', '$'):
                    self.update_tokens()

            except StopIteration:
                break

            # Create the next namelist
            self.update_tokens()
            g_name = self.token

            g_vars = Namelist()
            v_name = None

            # TODO: Edit `Namelist` to support case-insensitive `get` calls
            grp_patch = nml_patch.get(g_name.lower(), {})

            # Populate the namelist group
            while g_name:

                if self.token not in ('=', '%', '('):
                    self.update_tokens()

                # Set the next active variable
                if self.token in ('=', '(', '%'):

                    try:
                        v_name, v_values = self.parse_variable(
                            g_vars, patch_nml=grp_patch)
                    except ValueError:
                        nml_file.close()
                        if self.pfile:
                            self.pfile.close()
                        raise

                    if v_name in g_vars:
                        v_prior_values = g_vars[v_name]
                        v_values = merge_values(v_prior_values, v_values)

                    if v_name in g_vars and isinstance(g_vars[v_name], dict):
                        g_vars[v_name].update(v_values)
                    else:
                        g_vars[v_name] = v_values

                    # Deselect variable
                    v_name = None
                    v_values = []

                # Finalise namelist group
                if self.token in ('/', '&', '$'):

                    # Append any remaining patched variables
                    for v_name, v_val in grp_patch.items():
                        g_vars[v_name] = v_val
                        v_strs = nmls.var_strings(v_name, v_val)
                        for v_str in v_strs:
                            self.pfile.write('    {0}\n'.format(v_str))

                    # Append the grouplist to the namelist
                    if g_name in nmls:
                        g_update = nmls[g_name]

                        # Update to list of groups
                        if not isinstance(g_update, list):
                            g_update = [g_update]

                        g_update.append(g_vars)

                    else:
                        g_update = g_vars

                    nmls[g_name] = g_update

                    # Reset state
                    g_name, g_vars = None, None

            try:
                self.update_tokens()
            except StopIteration:
                break

        nml_file.close()
        if self.pfile:
            self.pfile.close()

        return nmls

    def parse_variable(self, parent, patch_nml=None):
        """Parse a variable and return its name and values."""

        if not patch_nml:
            patch_nml = Namelist()

        v_name = self.prior_token
        v_values = []

        # Patch state
        patch_values = None
        write_token = v_name not in patch_nml

        if self.token == '(':

            v_idx_bounds = self.parse_indices()
            v_idx = FIndex(v_idx_bounds)

            self.update_tokens()
        else:
            v_idx = None

        if self.token == '%':

            # Resolve the derived type

            if parent and v_name in parent:
                v_parent = parent[v_name]<commit_msg>Improved open file handling on parser exceptions<commit_after>"""f90nml.parser
   =============

   Fortran namelist parser and tokenizer to convert contents into a hierarchy
   of dicts containing intrinsic Python data types.

   :copyright: Copyright 2014 <NAME>, see AUTHORS for details.
   :license: Apache License, Version 2.0, see LICENSE for details.
"""
import copy
import shlex
from string import whitespace

from f90nml.fpy import pyfloat, pycomplex, pybool, pystr
from f90nml.namelist import Namelist
from f90nml.findex import FIndex


class Parser(object):
    """shlex-based Fortran namelist parser."""

    def __init__(self):

        # Token management
        self.tokens = None
        self.token = None
        self.prior_token = None

        # Patching
        self.pfile = None

        # Control flags
        self._row_major = False
        self._strict_logical = True

        # Configuration
        self.comment_tokens = '!'

    @property
    def row_major(self):
        """Return true if multidimensional arrays are in row-major format."""
        return self._row_major

    @row_major.setter
    def row_major(self, value):
        """Validate and set row-major format for multidimensional arrays."""

        if value is not None:
            if not isinstance(value, bool):
                raise ValueError(
                    'f90nml: error: row_major must be a logical value.')
            else:
                self._row_major = value

    @property
    def strict_logical(self):
        """Return true for strict logical value parsing."""
        return self._strict_logical

    @strict_logical.setter
    def strict_logical(self, value):
        """Validate and set the strict logical flag."""

        if value is not None:
            if not isinstance(value, bool):
                raise ValueError(
                    'f90nml: error: strict_logical must be a logical value.')
            else:
                self._strict_logical = value

    def read(self, nml_fname, nml_patch_in=None, patch_fname=None):
        """Parse a Fortran 90 namelist file and store the contents.

        >>> from f90nml.parser import Parser
        >>> parser = Parser()
        >>> data_nml = parser.read('data.nml')"""

        # Convert patch data to a Namelist object
        if nml_patch_in:
            if not isinstance(nml_patch_in, dict):
                raise ValueError('Input patch must be a dict or a Namelist.')

            nml_patch = copy.deepcopy(Namelist(nml_patch_in))

            if not patch_fname:
                patch_fname = nml_fname + '~'
            elif nml_fname == patch_fname:
                raise ValueError('f90nml: error: Patch filepath cannot be the '
                                 'same as the original filepath.')
            self.pfile = open(patch_fname, 'w')
        else:
            nml_patch = Namelist()

        try:
            nml_file = open(nml_fname, 'r')
            return self.readstream(nml_file, nml_patch)

        finally:
            # Close the unfinished files on any exceptions within readstream
            nml_file.close()
            if self.pfile:
                self.pfile.close()

    def readstream(self, nml_file, nml_patch):

        f90lex = shlex.shlex(nml_file)
        f90lex.whitespace = ''
        f90lex.wordchars += '.-+'       # Include floating point tokens
        if nml_patch:
            f90lex.commenters = ''
        else:
            f90lex.commenters = self.comment_tokens

        self.tokens = iter(f90lex)

        nmls = Namelist()

        # TODO: Replace "while True" with an update_token() iterator
        self.update_tokens(write_token=False)
        while True:
            try:
                # Check for classic group terminator
                if self.token == 'end':
                    self.update_tokens()

                # Ignore tokens outside of namelist groups
                while self.token not in ('&', '$'):
                    self.update_tokens()

            except StopIteration:
                break

            # Create the next namelist
            self.update_tokens()
            g_name = self.token

            g_vars = Namelist()
            v_name = None

            # TODO: Edit `Namelist` to support case-insensitive `get` calls
            grp_patch = nml_patch.get(g_name.lower(), {})

            # Populate the namelist group
            while g_name:

                if self.token not in ('=', '%', '('):
                    self.update_tokens()

                # Set the next active variable
                if self.token in ('=', '(', '%'):

                    v_name, v_values = self.parse_variable(g_vars,
                                                           patch_nml=grp_patch)

                    if v_name in g_vars:
                        v_prior_values = g_vars[v_name]
                        v_values = merge_values(v_prior_values, v_values)

                    if v_name in g_vars and isinstance(g_vars[v_name], dict):
                        g_vars[v_name].update(v_values)
                    else:
                        g_vars[v_name] = v_values

                    # Deselect variable
                    v_name = None
                    v_values = []

                # Finalise namelist group
                if self.token in ('/', '&', '$'):

                    # Append any remaining patched variables
                    for v_name, v_val in grp_patch.items():
                        g_vars[v_name] = v_val
                        v_strs = nmls.var_strings(v_name, v_val)
                        for v_str in v_strs:
                            self.pfile.write('    {0}\n'.format(v_str))

                    # Append the grouplist to the namelist
                    if g_name in nmls:
                        g_update = nmls[g_name]

                        # Update to list of groups
                        if not isinstance(g_update, list):
                            g_update = [g_update]

                        g_update.append(g_vars)

                    else:
                        g_update = g_vars

                    nmls[g_name] = g_update

                    # Reset state
                    g_name, g_vars = None, None

            try:
                self.update_tokens()
            except StopIteration:
                break

        return nmls

    def parse_variable(self, parent, patch_nml=None):
        """Parse a variable and return its name and values."""

        if not patch_nml:
            patch_nml = Namelist()

        v_name = self.prior_token
        v_values = []

        # Patch state
        patch_values = None
        write_token = v_name not in patch_nml

        if self.token == '(':

            v_idx_bounds = self.parse_indices()
            v_idx = FIndex(v_idx_bounds)

            self.update_tokens()
        else:
            v_idx = None

        if self.token == '%':

            # Resolve the derived type

            if parent and v_name in parent:
                v_parent = parent[v_name]
{'fraction_non_alphanumeric': 0.08019035442858158, 'fraction_numerical': 0.005256055117550963, 'mean_word_length': 1.4444444444444444, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6152531', 'n_tokens_mistral': 4018, 'n_tokens_neox': 3702, 'n_words': 1046}