<commit_before>###############################################################################
#
#   Package: NaturalDocs::SymbolTable
#
###############################################################################
#
#   A package that handles all the gory details of managing symbols.  It handles where they are defined, which files
#   reference them, if any are undefined or duplicated, and loading and saving them to a file.
#
#   Usage and Dependencies:
#
#       - <NaturalDocs::Settings>, <NaturalDocs::Languages>, and <NaturalDocs::Project> must be initialized before use.
#
#       - <LoadAndPurge()> must be called to initialize the package.  No other functions can be called beforehand.  Afterwards
#         the <Modification Functions> like <AddSymbol()> are available, but the symbol table still isn't fully resolved to the
#         changes in the source tree so the <Information Functions> are still out.
#
#       - <NaturalDocs::Parser::ParseForInformation()> must be called on all files that have changed to fully resolve the symbol
#         table.  Afterwards <PurgeResolvingInfo()> can be called and the <Information Functions> are available.
#
#       - <Save()> must be called to commit any changes to the symbol table back to disk.
#
###############################################################################

# This file is part of Natural Docs, which is Copyright  2003 <NAME>
# Natural Docs is licensed under the GPL


use NaturalDocs::SymbolTable::Symbol;
use NaturalDocs::SymbolTable::SymbolDefinition;
use NaturalDocs::SymbolTable::Reference;
use NaturalDocs::SymbolTable::File;
use NaturalDocs::SymbolTable::ReferenceTarget;
use NaturalDocs::SymbolTable::IndexElement;

use strict;
use integer;

package NaturalDocs::SymbolTable;


###############################################################################
# Group: Variables

#
#   hash: symbols
#
#   A hash of all symbols, both those that are defined and those which are simply potential interpretations of references.  The
#   keys are the symbol strings, generated by by <MakeSymbolString()>, and the values are
#   <NaturalDocs::SymbolTable::Symbol> objects.  Note that because references can have many interpretations, there will
#   probably be more symbols here that aren't defined than that are.
#
my %symbols;

#
#   hash: references
#
#   A hash of all references in the project.  The keys are the reference strings, generated by by <MakeReferenceString()>,
#   and the values are <NaturalDocs::SymbolTable::Reference> objects.
#
my %references;

#
#   hash: files
#
#   A hash of all the files that define symbols and references in the project.  The keys are the file names, and the values are
#   <NaturalDocs::SymbolTable::File> objects.
#
my %files;

#
#   object: watchedFile
#
#   A <NaturalDocs::SymbolTable::File> object of the file being watched for changes.  This is compared to the version in <files>
#   to see if anything was changed since the last parse.
#
my $watchedFile;

#
#   string: watchedFileName
#
#   The file name of the watched file, if any.  If there is no watched file, this will be undef.
#
my $watchedFileName;

#
#   hash: watchedFileSymbolDefinitions
#
#   A hashref of the symbol definition information for all the symbols in the watched file.  The keys are the symbol strings, and the
#   values are <NaturalDocs::SymbolTable::SymbolDefinition> objects.
#
my %watchedFileSymbolDefinitions;


#
#   hash: indexes;
#
#   A hash of generated symbol indexes.  The keys are <Topic Types> or *, and the values are sorted arrayrefs of
#   <NaturalDocs::SymbolTable::IndexElements>.  A key of * means the index is not limited to a topic type.
#
my %indexes;


#
#   hash: indexChanges
#
#   An existence hash of all the indexes that have changed.  The keys are the <Topic Types>.  Unlike <indexes>, there is no *
#   entry.  Simply check if there are any defined keys instead.
#
my %indexChanges;


###############################################################################
# Group: Files

#
#   File: NaturalDocs.sym
#
#   The storage file for the symbol table.
#
#   Format:
#
#       The beginning of the file is the application version that generated it.  Handle it using the text file functions in
#       <NaturalDocs::Version>.
#
#       > [app version]
#
#       The first stage of the file is for symbol definitions, analogous to <symbols>.  For this stage, each entry continues until it
#       reaches a blank line.  The first line is the symbol string, and each subsequent line is a tab separated list of the definition,
#       type, and prototype.  The first file on this list will be the one that has the global definition.  Undefined symbols are
#       not included here.
#
#       > [symbol string]
#       > [global definition file] tab [symbol type] tab [prototype]
#       > [definition file] tab [symbol type] tab [prototype]
#       > blank line
#
#       The two stages are separated by a blank line.  That works out to meaning a symbol definition followed by two blank lines.
#
#       > blank line
#
#       The second stage is for references, which is analogous to <references>.  For this stage, each is a pair of lines.  The first line
#       is the reference string, and the second is a tab separated list of the files that define it.  The interpretations will be
#       regenerated.
#
#       > [reference string]
#       > [definition file] tab [definition file] tab [definition file] ...
#
#   Revisions:
#
#       Prior to 0.95, the version line was 1.  Test for "1" instead of "1.0" to distinguish.  Other than that, the file format has not
#       changed since its public release.
#


###############################################################################
# Group: File Functions


#
#   Function: LoadAndPurge
#
#   Loads the symbol table from disk, and purges everything associated with files that no longer have Natural Docs content.
#
sub LoadAndPurge
    {
    my $line;
    my $fileIsOkay;

    if (!NaturalDocs::Settings::RebuildData() && open(SYMBOLTABLEFILEHANDLE, '<' . NaturalDocs::Project::SymbolTableFile()))
        {
        # Check if the version is okay.

        my $version = NaturalDocs::Version::FromTextFile(\*SYMBOLTABLEFILEHANDLE);

        # Currently, the file format hasn't changed between public versions, so any version <= our own is okay.

        if ($version <= NaturalDocs::Settings::AppVersion())
            {  $fileIsOkay = 1;  }
        else
            {  close(SYMBOLTABLEFILEHANDLE);  };
        };


    # If something's wrong with the file, we need to reparse everything.
    if (!$fileIsOkay)
        {
        NaturalDocs::Project::ReparseEverything();
        return;
        };


    # Symbol definitions

    while ($line = <SYMBOLTABLEFILEHANDLE>)
        {
        chomp($line);

        # Putting this in the while condition causes problems.
        if (!length $line)
            {  last;  };

        # [symbol string]

        my $symbolString = $line;
        my $symbolObject = NaturalDocs::SymbolTable::Symbol::New();

        $line = <SYMBOLTABLEFILEHANDLE>;
        chomp($line);

        while (length $line)
            {
            # [definition] tab [type] tab [prototype]

            my ($file, $type, $prototype) = split(/\t/, $line);
            $symbolObject->AddDefinition($file, $type, $prototype);

            if (!exists $files{$file})
                {  $files{$file} = NaturalDocs::SymbolTable::File::New();  };

            $files{$file}->AddSymbol($symbolString);

            $line = <SYMBOLTABLEFILEHANDLE>;
            chomp($line);
            };<commit_msg>Make sure prototype is undef and not an empty string.<commit_after>###############################################################################
#
#   Package: NaturalDocs::SymbolTable
#
###############################################################################
#
#   A package that handles all the gory details of managing symbols.  It handles where they are defined, which files
#   reference them, if any are undefined or duplicated, and loading and saving them to a file.
#
#   Usage and Dependencies:
#
#       - <NaturalDocs::Settings>, <NaturalDocs::Languages>, and <NaturalDocs::Project> must be initialized before use.
#
#       - <LoadAndPurge()> must be called to initialize the package.  No other functions can be called beforehand.  Afterwards
#         the <Modification Functions> like <AddSymbol()> are available, but the symbol table still isn't fully resolved to the
#         changes in the source tree so the <Information Functions> are still out.
#
#       - <NaturalDocs::Parser::ParseForInformation()> must be called on all files that have changed to fully resolve the symbol
#         table.  Afterwards <PurgeResolvingInfo()> can be called and the <Information Functions> are available.
#
#       - <Save()> must be called to commit any changes to the symbol table back to disk.
#
###############################################################################

# This file is part of Natural Docs, which is Copyright  2003 <NAME>
# Natural Docs is licensed under the GPL


use NaturalDocs::SymbolTable::Symbol;
use NaturalDocs::SymbolTable::SymbolDefinition;
use NaturalDocs::SymbolTable::Reference;
use NaturalDocs::SymbolTable::File;
use NaturalDocs::SymbolTable::ReferenceTarget;
use NaturalDocs::SymbolTable::IndexElement;

use strict;
use integer;

package NaturalDocs::SymbolTable;


###############################################################################
# Group: Variables

#
#   hash: symbols
#
#   A hash of all symbols, both those that are defined and those which are simply potential interpretations of references.  The
#   keys are the symbol strings, generated by by <MakeSymbolString()>, and the values are
#   <NaturalDocs::SymbolTable::Symbol> objects.  Note that because references can have many interpretations, there will
#   probably be more symbols here that aren't defined than that are.
#
my %symbols;

#
#   hash: references
#
#   A hash of all references in the project.  The keys are the reference strings, generated by by <MakeReferenceString()>,
#   and the values are <NaturalDocs::SymbolTable::Reference> objects.
#
my %references;

#
#   hash: files
#
#   A hash of all the files that define symbols and references in the project.  The keys are the file names, and the values are
#   <NaturalDocs::SymbolTable::File> objects.
#
my %files;

#
#   object: watchedFile
#
#   A <NaturalDocs::SymbolTable::File> object of the file being watched for changes.  This is compared to the version in <files>
#   to see if anything was changed since the last parse.
#
my $watchedFile;

#
#   string: watchedFileName
#
#   The file name of the watched file, if any.  If there is no watched file, this will be undef.
#
my $watchedFileName;

#
#   hash: watchedFileSymbolDefinitions
#
#   A hashref of the symbol definition information for all the symbols in the watched file.  The keys are the symbol strings, and the
#   values are <NaturalDocs::SymbolTable::SymbolDefinition> objects.
#
my %watchedFileSymbolDefinitions;


#
#   hash: indexes;
#
#   A hash of generated symbol indexes.  The keys are <Topic Types> or *, and the values are sorted arrayrefs of
#   <NaturalDocs::SymbolTable::IndexElements>.  A key of * means the index is not limited to a topic type.
#
my %indexes;


#
#   hash: indexChanges
#
#   An existence hash of all the indexes that have changed.  The keys are the <Topic Types>.  Unlike <indexes>, there is no *
#   entry.  Simply check if there are any defined keys instead.
#
my %indexChanges;


###############################################################################
# Group: Files

#
#   File: NaturalDocs.sym
#
#   The storage file for the symbol table.
#
#   Format:
#
#       The beginning of the file is the application version that generated it.  Handle it using the text file functions in
#       <NaturalDocs::Version>.
#
#       > [app version]
#
#       The first stage of the file is for symbol definitions, analogous to <symbols>.  For this stage, each entry continues until it
#       reaches a blank line.  The first line is the symbol string, and each subsequent line is a tab separated list of the definition,
#       type, and prototype.  The first file on this list will be the one that has the global definition.  Undefined symbols are
#       not included here.
#
#       > [symbol string]
#       > [global definition file] tab [symbol type] tab [prototype]
#       > [definition file] tab [symbol type] tab [prototype]
#       > blank line
#
#       The two stages are separated by a blank line.  That works out to meaning a symbol definition followed by two blank lines.
#
#       > blank line
#
#       The second stage is for references, which is analogous to <references>.  For this stage, each is a pair of lines.  The first line
#       is the reference string, and the second is a tab separated list of the files that define it.  The interpretations will be
#       regenerated.
#
#       > [reference string]
#       > [definition file] tab [definition file] tab [definition file] ...
#
#   Revisions:
#
#       Prior to 0.95, the version line was 1.  Test for "1" instead of "1.0" to distinguish.  Other than that, the file format has not
#       changed since its public release.
#


###############################################################################
# Group: File Functions


#
#   Function: LoadAndPurge
#
#   Loads the symbol table from disk, and purges everything associated with files that no longer have Natural Docs content.
#
sub LoadAndPurge
    {
    my $line;
    my $fileIsOkay;

    if (!NaturalDocs::Settings::RebuildData() && open(SYMBOLTABLEFILEHANDLE, '<' . NaturalDocs::Project::SymbolTableFile()))
        {
        # Check if the version is okay.

        my $version = NaturalDocs::Version::FromTextFile(\*SYMBOLTABLEFILEHANDLE);

        # Currently, the file format hasn't changed between public versions, so any version <= our own is okay.

        if ($version <= NaturalDocs::Settings::AppVersion())
            {  $fileIsOkay = 1;  }
        else
            {  close(SYMBOLTABLEFILEHANDLE);  };
        };


    # If something's wrong with the file, we need to reparse everything.
    if (!$fileIsOkay)
        {
        NaturalDocs::Project::ReparseEverything();
        return;
        };


    # Symbol definitions

    while ($line = <SYMBOLTABLEFILEHANDLE>)
        {
        chomp($line);

        # Putting this in the while condition causes problems.
        if (!length $line)
            {  last;  };

        # [symbol string]

        my $symbolString = $line;
        my $symbolObject = NaturalDocs::SymbolTable::Symbol::New();

        $line = <SYMBOLTABLEFILEHANDLE>;
        chomp($line);

        while (length $line)
            {
            # [definition] tab [type] tab [prototype]

            my ($file, $type, $prototype) = split(/\t/, $line);

            if (!$prototype)
                {  $prototype = undef;  };

            $symbolObject->AddDefinition($file, $type, $prototype);

            if (!exists $files{$file})
                {  $files{$file} = NaturalDocs::SymbolTable::File::New();  };

            $files{$file}->AddSymbol($symbolString);

            $line = <SYMBOLTABLEFILEHANDLE>;
            chomp($line);
            };
{'fraction_non_alphanumeric': 0.14680989583333334, 'fraction_numerical': 0.0015625, 'mean_word_length': 3.3564946114577423, 'pattern_counts': {'":': 0, '<': 69, '<?xml version=': 0, '>': 83, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7549470', 'n_tokens_mistral': 4150, 'n_tokens_neox': 3951, 'n_words': 1653}