<commit_before>#!/usr/bin/perl -w

use strict;
use warnings;

use FindBin '$Bin';

use lib $Bin;

use CodeSearchHelper ();

use Getopt::Long 'GetOptions';

use File::Slurp 'slurp';

use YAML 'Load', 'Dump';

use List::MoreUtils 'uniq';

use PPI ();


my $db_file = '';
my $sub_name = '';
my $include_files_info = '';

GetOptions(
    'db=s' => \$db_file,
    'name=s' => \$sub_name,
    'files' => \$include_files_info,
);

unless(
    ( -f $db_file )
    && $sub_name
) {

    die 'Not enough arguments';
}

my $db = Load( scalar( slurp( $db_file ) ) );

unless( exists( $db -> { $sub_name } ) ) {

    die 'No such call in DB';
}

my %output    = ();
my @sub_names = ( $sub_name );
my %files     = ();

while( defined( my $sub_name = shift( @sub_names ) ) ) {

    next unless( exists( $db -> { $sub_name } ) );

    my $local_output = $output{ $sub_name } //= {};

    foreach my $file ( @{ $db -> { $sub_name } } ) {

        my $doc = PPI::Document -> new( $file );

        CodeSearchHelper::scan_calls( $doc, sub {

            my ( $name_str, $name ) = @_;

            return if( $name_str ne $sub_name );

            while( defined( my $parent = $name -> parent() ) ) {

                if( ref( $parent ) eq 'PPI::Statement::Sub' ) {

                    my $parent_name = $parent -> name();

                    unless( exists( $output{ $parent_name } ) ) {

                        $output{ $parent_name } = $local_output -> { $parent_name } = {};

                        push( @sub_names, $parent_name );
                    }

                    push( @{ $files{ $parent_name } }, $file );

                    last;
                }

                $name = $parent;
            }
        } );
    }
}

if( $include_files_info ) {

    while( my ( $name, $list ) = each( %files ) ) {

        $files{ $name } = [ uniq( @$list ) ];
    }

    my @nodes = ( $output{ $sub_name } );

    while( defined( my $node = shift( @nodes ) ) ) {

        while( my ( $name, $list ) = each( %$node ) ) {

            %$list = (
                files => $files{ $name },
                calls => { %$list },
            );

            push( @nodes, $list -> { 'calls' } );
        }
    }
<commit_msg>trying to not lose stuff<commit_after>#!/usr/bin/perl -w

use strict;
use warnings;

use FindBin '$Bin';

use lib $Bin;

use CodeSearchHelper ();

use Getopt::Long 'GetOptions';

use File::Slurp 'slurp';

use YAML 'Load', 'Dump';

use List::MoreUtils 'uniq';

use PPI ();


my $db_file = '';
my $sub_name = '';
my $include_files_info = '';

GetOptions(
    'db=s' => \$db_file,
    'name=s' => \$sub_name,
    'files' => \$include_files_info,
);

unless(
    ( -f $db_file )
    && $sub_name
) {

    die 'Not enough arguments';
}

my $db = Load( scalar( slurp( $db_file ) ) );

unless( exists( $db -> { $sub_name } ) ) {

    die 'No such call in DB';
}

my %output    = ();
my @sub_names = ( $sub_name );
my %files     = ();

while( defined( my $sub_name = shift( @sub_names ) ) ) {

    next unless( exists( $db -> { $sub_name } ) );

    my $local_output = $output{ $sub_name } //= {};

    foreach my $file ( @{ $db -> { $sub_name } } ) {

        my $doc = PPI::Document -> new( $file );

        CodeSearchHelper::scan_calls( $doc, sub {

            my ( $name_str, $name ) = @_;

            return if( $name_str ne $sub_name );

            while( defined( my $parent = $name -> parent() ) ) {

                if( ref( $parent ) eq 'PPI::Statement::Sub' ) {

                    my $parent_name = $parent -> name();

                    unless( exists( $output{ $parent_name } ) ) {

                        push( @sub_names, $parent_name );
                    }

                    $output{ $parent_name } = $local_output -> { $parent_name } //= {};

                    push( @{ $files{ '' . $local_output -> { $parent_name } } }, $file );

                    last;
                }

                $name = $parent;
            }
        } );
    }
}

if( $include_files_info ) {

    while( my ( $name, $list ) = each( %files ) ) {

        $files{ $name } = [ uniq( @$list ) ];
    }

    my @nodes = ( $output{ $sub_name } );

    while( defined( my $node = shift( @nodes ) ) ) {

        while( my ( $name, $list ) = each( %$node ) ) {

            %$list = (
                files => $files{ '' . $list },
                calls => { %$list },
            );

            push( @nodes, $list -> { 'calls' } );
        }
    }

{'fraction_non_alphanumeric': 0.18110412926391384, 'fraction_numerical': 0.0, 'mean_word_length': 1.7077764277035237, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4504919', 'n_tokens_mistral': 1594, 'n_tokens_neox': 1503, 'n_words': 365}