<commit_before>#!/usr/bin/env perl

$VERSION = "0.1";

use strict;
use warnings;
use Data::Dumper;
use Digest::MD5 qw(md5_hex);

my @modules;
my %options = ();
my $cur_module;
my $example_language = "lua";

my %languages = (
    c => {
        start  => qr/^\s*\/\*\*\*(?:\s*|(\s+\S.+\s*))$/,
        end    => qr/^\s*\*+\/\s*$/,
        filter => qr/^(?:\s*\*+\s?)?(\s*[^*].+)\s*$/,
    },
    lua => {
        start  => qr/^\s*\--\[\[\[\s*$/,
        end    => qr/^\s*--\]\]\s*/,
        filter => qr/^(?:\s*--\s)?(\s*\S.+)\s*$/,
    },
);

my $function_re = qr/^\s*\@(function|fn|method)\s*(\S.+)$/oi;
my $module_re = qr/^\s*\@(?:module|file)\s*(\S.+)$/oi;

my $language;

sub print_module_markdown {
    my ( $mname, $m ) = @_;

    my $idline = $options{g} ? "" : " {#$m->{'id'}}";
    print <<EOD;
## Module `$mname`$idline

$m->{'data'}
EOD
    if ( $m->{'example'} ) {
        print <<EOD;

Example:

~~~$m->{'example_language'}
$m->{'example'}
~~~
EOD
    }

    sub print_func {
        my ($f) = @_;

        my $name = $f->{'name'};
        my $id   = $f->{'id'};
        if ($f->{'brief'}) {
            print "> [`$name`](#$id): ". $f->{'brief'} . "\n\n";
        }
        else {
            print "> [`$name`](#$id)\n\n";
        }
    }

    print "\n###Brief content:\n\n";
    print "**Functions**:\n\n";
    foreach ( @{ $m->{'functions'} } ) {
        print_func($_);
    }
    print "\n\n**Methods**:\n\n";
    foreach ( @{ $m->{'methods'} } ) {
        print_func($_);
    }
}

sub print_function_markdown {
    my ( $type, $fname, $f ) = @_;

    my $idline = $options{g} ? "" : " {#$f->{'id'}}";
    print <<EOD;
### $type `$fname`$idline

$f->{'data'}
EOD
    print "\n**Parameters:**\n\n";
    if ( $f->{'params'} && scalar @{ $f->{'params'} } > 0 ) {
        foreach ( @{ $f->{'params'} } ) {
            if ( $_->{'type'} ) {
                print
                    "- `$_->{'name'} \{$_->{'type'}\}`: $_->{'description'}\n";
            }
            else {
                print "- `$_->{'name'}`: $_->{'description'}\n";
            }
        }
    }
    else {
        print "\tnothing\n";
    }
    print "\n**Returns:**\n\n";
    if ( $f->{'return'} && $f->{'return'}->{'description'} ) {
        $_ = $f->{'return'};
        if ( $_->{'type'} ) {
            print "- `\{$_->{'type'}\}`: $_->{'description'}\n";
        }
        else {
            print "- $_->{'description'}\n";
        }
    }
    else {
        print "\tnothing\n";
    }
    if ( $f->{'example'} ) {
        print <<EOD;

Example:

~~~$f->{'example_language'}
$f->{'example'}
~~~
EOD
    }
}

sub print_markdown {
    for my $m (@modules) {
        my $mname = $m->{name};
        print_module_markdown( $mname, $m );

        print
            "\n## Functions\n\nThe module `$mname` defines the following functions.\n\n";
        foreach ( @{ $m->{'functions'} } ) {
            print_function_markdown( "Function", $_->{'name'}, $_ );
            print "\nBack to [module description](#$m->{'id'}).\n\n";

        }
        print
            "\n## Methods\n\nThe module `$mname` defines the following methods.\n\n";
        foreach ( @{ $m->{'methods'} } ) {
            print_function_markdown( "Method", $_->{'name'}, $_ );
            print "\nBack to [module description](#$m->{'id'}).\n\n";

        }
        print "\nBack to [top](#).\n\n";
    }
}

sub make_id {
    my ( $name, $prefix ) = @_;

    if ( !$prefix ) {
        $prefix = "f";
    }
    if ( !$options{g} ) {

        # Kramdown/pandoc version of ID's
        $name =~ /^(\S+).*$/;
        return substr( substr( $prefix, 0, 1 ) . md5_hex($1), 0, 6 );
    }
    else {
        my $input = lc $prefix . "-" . $name;
        my $id = join '-', split /\s+/, $input;
        $id =~ s/[^\w_-]+//g;
        return $id;
    }
}

sub substitute_data_keywords {
    my ($line) = @_;

    if ( $line =~ /^.*\@see\s+(\S+)\s*.*$/ ) {
        my $name = $1;
        my $id   = make_id($name);
        return $line =~ s/\@see\s+\S+/[`$name`](#$id)/r;<commit_msg>Do not print absend methods/functions.<commit_after>#!/usr/bin/env perl

$VERSION = "0.1";

use strict;
use warnings;
use Data::Dumper;
use Digest::MD5 qw(md5_hex);

my @modules;
my %options = ();
my $cur_module;
my $example_language = "lua";

my %languages = (
    c => {
        start  => qr/^\s*\/\*\*\*(?:\s*|(\s+\S.+\s*))$/,
        end    => qr/^\s*\*+\/\s*$/,
        filter => qr/^(?:\s*\*+\s?)?(\s*[^*].+)\s*$/,
    },
    lua => {
        start  => qr/^\s*\--\[\[\[\s*$/,
        end    => qr/^\s*--\]\]\s*/,
        filter => qr/^(?:\s*--\s)?(\s*\S.+)\s*$/,
    },
);

my $function_re = qr/^\s*\@(function|fn|method)\s*(\S.+)$/oi;
my $module_re = qr/^\s*\@(?:module|file)\s*(\S.+)$/oi;

my $language;

sub print_module_markdown {
    my ( $mname, $m ) = @_;

    my $idline = $options{g} ? "" : " {#$m->{'id'}}";
    print <<EOD;
## Module `$mname`$idline

$m->{'data'}
EOD
    if ( $m->{'example'} ) {
        print <<EOD;

Example:

~~~$m->{'example_language'}
$m->{'example'}
~~~
EOD
    }

    sub print_func {
        my ($f) = @_;

        my $name = $f->{'name'};
        my $id   = $f->{'id'};
        if ($f->{'brief'}) {
            print "> [`$name`](#$id): ". $f->{'brief'} . "\n\n";
        }
        else {
            print "> [`$name`](#$id)\n\n";
        }
    }

    print "\n###Brief content:\n\n";
    if (scalar(@{ $m->{'functions'} }) > 0) {
        print "**Functions**:\n\n";
        foreach ( @{ $m->{'functions'} } ) {
        print_func($_);
        }
    }
    if (scalar(@{ $m->{'methods'} }) > 0) {
        print "\n\n**Methods**:\n\n";
        foreach (@{ $m->{'methods'} }) {
            print_func($_);
        }
    }
}

sub print_function_markdown {
    my ( $type, $fname, $f ) = @_;

    my $idline = $options{g} ? "" : " {#$f->{'id'}}";
    print <<EOD;
### $type `$fname`$idline

$f->{'data'}
EOD
    print "\n**Parameters:**\n\n";
    if ( $f->{'params'} && scalar @{ $f->{'params'} } > 0 ) {
        foreach ( @{ $f->{'params'} } ) {
            if ( $_->{'type'} ) {
                print
                    "- `$_->{'name'} \{$_->{'type'}\}`: $_->{'description'}\n";
            }
            else {
                print "- `$_->{'name'}`: $_->{'description'}\n";
            }
        }
    }
    else {
        print "\tnothing\n";
    }
    print "\n**Returns:**\n\n";
    if ( $f->{'return'} && $f->{'return'}->{'description'} ) {
        $_ = $f->{'return'};
        if ( $_->{'type'} ) {
            print "- `\{$_->{'type'}\}`: $_->{'description'}\n";
        }
        else {
            print "- $_->{'description'}\n";
        }
    }
    else {
        print "\tnothing\n";
    }
    if ( $f->{'example'} ) {
        print <<EOD;

Example:

~~~$f->{'example_language'}
$f->{'example'}
~~~
EOD
    }
}

sub print_markdown {
    for my $m (@modules) {
        my $mname = $m->{name};
        print_module_markdown( $mname, $m );

        if (scalar(@{ $m->{'functions'} }) > 0) {
            print
                "\n## Functions\n\nThe module `$mname` defines the following functions.\n\n";
            foreach (@{ $m->{'functions'} }) {
                print_function_markdown( "Function", $_->{'name'}, $_ );
                print "\nBack to [module description](#$m->{'id'}).\n\n";

            }
        }

        if (scalar(@{ $m->{'methods'} }) > 0) {
            print
                "\n## Methods\n\nThe module `$mname` defines the following methods.\n\n";
            foreach (@{ $m->{'methods'} }) {
                print_function_markdown( "Method", $_->{'name'}, $_ );
                print "\nBack to [module description](#$m->{'id'}).\n\n";

            }
        }

        print "\nBack to [top](#).\n\n";
    }
}

sub make_id {
    my ( $name, $prefix ) = @_;

    if ( !$prefix ) {
        $prefix = "f";
    }
    if ( !$options{g} ) {

        # Kramdown/pandoc version of ID's
        $name =~ /^(\S+).*$/;
        return substr( substr( $prefix, 0, 1 ) . md5_hex($1), 0, 6 );
    }
    else {
        my $input = lc $prefix . "-" . $name;
        my $id = join '-', split /\s+/, $input;
        $id =~ s/[^\w_-]+//g;
        return $id;
    }
}

sub substitute_data_keywords {
    my ($line) = @_;

    if ( $line =~ /^.*\@see\s+(\S+)\s*.*$/ ) {
        my $name = $1;
        my $id   = make_id($name);
        return $line =~ s/\@see\s+\S+/[`$name`](#$id)/r;
{'fraction_non_alphanumeric': 0.2738876297268281, 'fraction_numerical': 0.0033400930454491233, 'mean_word_length': 1.7036439858110286, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 113, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7002438', 'n_tokens_mistral': 3310, 'n_tokens_neox': 3160, 'n_words': 571}