<commit_before>###############################################################################
#
#   Package: NaturalDocs::Parser
#
###############################################################################
#
#   A package that coordinates source file parsing between the <NaturalDocs::Languages::Base>-derived objects and its own
#   sub-packages such as <NaturalDocs::Parser::Native>.  Also handles sending symbols to <NaturalDocs::SymbolTable> and
#   other generic topic processing.
#
#   Usage and Dependencies:
#
#       - Prior to use, <NaturalDocs::Settings>, <NaturalDocs::Languages>, <NaturalDocs::Project>, <NaturalDocs::SymbolTable>,
#         and <NaturalDocs::ClassHierarchy> must be initialized.  <NaturalDocs::SymbolTable> and <NaturalDocs::ClassHierarchy>
#         do not have to be fully resolved.
#
#       - Aside from that, the package is ready to use right away.  It does not have its own initialization function.
#
###############################################################################

# This file is part of Natural Docs, which is Copyright (C) 2003-2005 <NAME>
# Natural Docs is licensed under the GPL

use NaturalDocs::Parser::ParsedTopic;
use NaturalDocs::Parser::Native;
use NaturalDocs::Parser::JavaDoc;

use strict;
use integer;

package NaturalDocs::Parser;



###############################################################################
# Group: Variables


#
#   var: sourceFile
#
#   The source <FileName> currently being parsed.
#
my $sourceFile;

#
#   var: language
#
#   The language object for the file, derived from <NaturalDocs::Languages::Base>.
#
my $language;

#
#   Array: parsedFile
#
#   An array of <NaturalDocs::Parser::ParsedTopic> objects.
#
my @parsedFile;


#
#   bool: parsingForInformation
#   Whether <ParseForInformation()> was called.  If false, then <ParseForBuild()> was called.
#
my $parsingForInformation;



###############################################################################
# Group: Functions

#
#   Function: ParseForInformation
#
#   Parses the input file for information.  Will update the information about the file in <NaturalDocs::SymbolTable> and
#   <NaturalDocs::Project>.
#
#   Parameters:
#
#       file - The <FileName> to parse.
#
sub ParseForInformation #(file)
    {
    my ($self, $file) = @_;
    $sourceFile = $file;

    $parsingForInformation = 1;

    # Watch this parse so we detect any changes.
    NaturalDocs::SymbolTable->WatchFileForChanges($sourceFile);
    NaturalDocs::ClassHierarchy->WatchFileForChanges($sourceFile);

    my $defaultMenuTitle = $self->Parse();

    foreach my $topic (@parsedFile)
        {
        # Add a symbol for the topic.

        my $type = $topic->Type();
        if ($type eq ::TOPIC_ENUMERATION())
            {  $type = ::TOPIC_TYPE();  };

        NaturalDocs::SymbolTable->AddSymbol($topic->Symbol(), $sourceFile, $type,
                                                                   $topic->Prototype(), $topic->Summary());


        # You can't put the function call directly in a while with a regex.  It has to sit in a variable to work.
        my $body = $topic->Body();


        # If it's a list or enum topic, add a symbol for each description list entry.

        if ($topic->IsList() || $topic->Type() eq ::TOPIC_ENUMERATION())
            {
            # We'll hijack the enum constants to apply to non-enum behavior too.
            my $behavior;

            if ($topic->Type() eq ::TOPIC_ENUMERATION())
                {
                $type = ::TOPIC_CONSTANT();
                $behavior = $language->EnumValues();
                }
            elsif (NaturalDocs::Topics->TypeInfo($topic->Type())->Scope() == ::SCOPE_ALWAYS_GLOBAL())
                {
                $behavior = ::ENUM_GLOBAL();
                }
            else
                {
                $behavior = ::ENUM_UNDER_PARENT();
                };

            while ($body =~ /<ds>([^<]+)<\/ds><dd>(.*?)<\/dd>/g)
                {
                my ($listTextSymbol, $listSummary) = ($1, $2);

                $listTextSymbol = NaturalDocs::NDMarkup->RestoreAmpChars($listTextSymbol);
                my $listSymbol = NaturalDocs::SymbolString->FromText($listTextSymbol);

                if ($behavior == ::ENUM_UNDER_PARENT())
                    {  $listSymbol = NaturalDocs::SymbolString->Join($topic->Package(), $listSymbol);  }
                elsif ($behavior == ::ENUM_UNDER_TYPE())
                    {  $listSymbol = NaturalDocs::SymbolString->Join($topic->Symbol(), $listSymbol);  };

                NaturalDocs::SymbolTable->AddSymbol($listSymbol, $sourceFile, $type, undef,
                                                                           $self->GetSummaryFromDescriptionList($listSummary));
                };
            };


        # Add references in the topic.

        while ($body =~ /<link>([^<]+)<\/link>/g)
            {
            my $linkText = NaturalDocs::NDMarkup->RestoreAmpChars($1);
            my $linkSymbol = NaturalDocs::SymbolString->FromText($linkText);
<commit_msg>Update to support NDMarkup changes.<commit_after>###############################################################################
#
#   Package: NaturalDocs::Parser
#
###############################################################################
#
#   A package that coordinates source file parsing between the <NaturalDocs::Languages::Base>-derived objects and its own
#   sub-packages such as <NaturalDocs::Parser::Native>.  Also handles sending symbols to <NaturalDocs::SymbolTable> and
#   other generic topic processing.
#
#   Usage and Dependencies:
#
#       - Prior to use, <NaturalDocs::Settings>, <NaturalDocs::Languages>, <NaturalDocs::Project>, <NaturalDocs::SymbolTable>,
#         and <NaturalDocs::ClassHierarchy> must be initialized.  <NaturalDocs::SymbolTable> and <NaturalDocs::ClassHierarchy>
#         do not have to be fully resolved.
#
#       - Aside from that, the package is ready to use right away.  It does not have its own initialization function.
#
###############################################################################

# This file is part of Natural Docs, which is Copyright (C) 2003-2005 <NAME>
# Natural Docs is licensed under the GPL

use NaturalDocs::Parser::ParsedTopic;
use NaturalDocs::Parser::Native;
use NaturalDocs::Parser::JavaDoc;

use strict;
use integer;

package NaturalDocs::Parser;



###############################################################################
# Group: Variables


#
#   var: sourceFile
#
#   The source <FileName> currently being parsed.
#
my $sourceFile;

#
#   var: language
#
#   The language object for the file, derived from <NaturalDocs::Languages::Base>.
#
my $language;

#
#   Array: parsedFile
#
#   An array of <NaturalDocs::Parser::ParsedTopic> objects.
#
my @parsedFile;


#
#   bool: parsingForInformation
#   Whether <ParseForInformation()> was called.  If false, then <ParseForBuild()> was called.
#
my $parsingForInformation;



###############################################################################
# Group: Functions

#
#   Function: ParseForInformation
#
#   Parses the input file for information.  Will update the information about the file in <NaturalDocs::SymbolTable> and
#   <NaturalDocs::Project>.
#
#   Parameters:
#
#       file - The <FileName> to parse.
#
sub ParseForInformation #(file)
    {
    my ($self, $file) = @_;
    $sourceFile = $file;

    $parsingForInformation = 1;

    # Watch this parse so we detect any changes.
    NaturalDocs::SymbolTable->WatchFileForChanges($sourceFile);
    NaturalDocs::ClassHierarchy->WatchFileForChanges($sourceFile);

    my $defaultMenuTitle = $self->Parse();

    foreach my $topic (@parsedFile)
        {
        # Add a symbol for the topic.

        my $type = $topic->Type();
        if ($type eq ::TOPIC_ENUMERATION())
            {  $type = ::TOPIC_TYPE();  };

        NaturalDocs::SymbolTable->AddSymbol($topic->Symbol(), $sourceFile, $type,
                                                                   $topic->Prototype(), $topic->Summary());


        # You can't put the function call directly in a while with a regex.  It has to sit in a variable to work.
        my $body = $topic->Body();


        # If it's a list or enum topic, add a symbol for each description list entry.

        if ($topic->IsList() || $topic->Type() eq ::TOPIC_ENUMERATION())
            {
            # We'll hijack the enum constants to apply to non-enum behavior too.
            my $behavior;

            if ($topic->Type() eq ::TOPIC_ENUMERATION())
                {
                $type = ::TOPIC_CONSTANT();
                $behavior = $language->EnumValues();
                }
            elsif (NaturalDocs::Topics->TypeInfo($topic->Type())->Scope() == ::SCOPE_ALWAYS_GLOBAL())
                {
                $behavior = ::ENUM_GLOBAL();
                }
            else
                {
                $behavior = ::ENUM_UNDER_PARENT();
                };

            while ($body =~ /<ds>([^<]+)<\/ds><dd>(.*?)<\/dd>/g)
                {
                my ($listTextSymbol, $listSummary) = ($1, $2);

                $listTextSymbol = NaturalDocs::NDMarkup->RestoreAmpChars($listTextSymbol);
                my $listSymbol = NaturalDocs::SymbolString->FromText($listTextSymbol);

                if ($behavior == ::ENUM_UNDER_PARENT())
                    {  $listSymbol = NaturalDocs::SymbolString->Join($topic->Package(), $listSymbol);  }
                elsif ($behavior == ::ENUM_UNDER_TYPE())
                    {  $listSymbol = NaturalDocs::SymbolString->Join($topic->Symbol(), $listSymbol);  };

                NaturalDocs::SymbolTable->AddSymbol($listSymbol, $sourceFile, $type, undef,
                                                                           $self->GetSummaryFromDescriptionList($listSummary));
                };
            };


        # Add references in the topic.

        while ($body =~ /<link(?:[^>]*)>([^<]+)<\/link>/g)
            {
            my $linkText = NaturalDocs::NDMarkup->RestoreAmpChars($1);
            my $linkSymbol = NaturalDocs::SymbolString->FromText($linkText);

{'fraction_non_alphanumeric': 0.19718863658704414, 'fraction_numerical': 0.0023591860808021233, 'mean_word_length': 2.6969476744186047, 'pattern_counts': {'":': 0, '<': 57, '<?xml version=': 0, '>': 106, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4140046', 'n_tokens_mistral': 2882, 'n_tokens_neox': 2663, 'n_words': 700}