<commit_before>##
##  ComponentJS -- Component System for JavaScript <http://componentjs.com>
##  Copyright (c) 2009-2013 <NAME> <http://engelschall.com>
##
##  This Source Code Form is subject to the terms of the Mozilla Public
##  License, v. 2.0. If a copy of the MPL was not distributed with this
##  file, You can obtain one at http://mozilla.org/MPL/2.0/.
##

use IO::File;

#   command-line arguments
my ($filename_txt, $filename_tmpl, $filename_html, $version) = @ARGV;

#   read textual description and template
my $fp = new IO::File "<$filename_txt"; $txt = ""; $txt .= $_ while (<$fp>); $fp->close();
$fp = new IO::File "<$filename_tmpl"; $tmpl = ""; $tmpl .= $_ while (<$fp>); $fp->close();

#   replace version information
$tmpl =~ s/0\.0\.0/$version/s;

#   start generating
my $html_spec = "";
my $html_navi = "";

#   duplication removal cache
my $cache = {};
my $defined = {};

#   remove comments
$txt =~ s/^#.*$//mg;

#   parse first-level structure
$txt =~ s/
    (?<=\n)([A-Z][^\n]+)\n  # headline
    ---+[ \t]*\n            # underlining
    (.+?\n)                 # body
    (?= [A-Z][^\n]+?\n      # following headline
        ---+[ \t]*\n        # corresponding underlining
    |   $             )     # or at end file
/
    parse2($1, $2), ''
/sgex;

#   process first-level structure
sub parse2 {
    my ($title, $body) = @_;

    #   generate body headline
    $html_spec .= "<h2>" . mklink(1, $title) . "</h2>\n";

    #   start navigation entry
    $html_navi .= "<h2>" . mklink(0, $title) . "</h2>\n";
    $html_navi .= "<ul>\n";

    #   parse second-level structure (part 1)
    $body =~ s/
        ^(.+?\n)           # intro paragraph
        (?=\n-[ \t]+\S+)   # start of first function
    /
        $html_spec .= "<div class=\"intro\">" . addpara(conv(0, $1)) . "<\/div>", ''
    /sex;
    sub addpara {
        my ($txt) = @_;
        $txt =~ s/(\n).[ \t]+([^\n]*(?:\n[ \t]+[^\n]+)*)/$1<li>$2<\/li>/sg;
        $txt =~ s/(\n\n)(<li>)/$1<ul>$2/sg;
        $txt =~ s/(<\/li>)(\n\n)/$1<\/ul>$2/sg;
        $txt =~ s/\n{2,}/<p\/>\n/sg;
        return $txt;
    }

    #   parse second-level structure (part 2)
    $body =~ s/
        (?<= \n)
        -[ \t]+(\S+.+?)    # function start
        (?= -[ \t]+\S+     # start of next function
        |   $         )    # or end of file
    /
        parse3($title, $1), ''
    /sgex;

    #   process third-level structure
    sub parse3 {
        my ($title, $body) = @_;

        $html_spec .= "<ul>\n";

        #   parse forth-level structure
        $body =~ s/
            ^
            (.+?)\n   #  synopsis
            \n        #  blank line
            (.+)      #  body
            $
        /
            parse4($title, $1, $2), ''
        /sex;

        #   process forth-level structure
        sub parse4 {
            my ($title, $synopsis, $body) = @_;

            $html_spec .= "<li>";

            #   parse fifth-level structure (part 1)
            my $txt = $synopsis;
            $txt =~ s/
                M<(.+?)>
            /
                parse5a($title, $1), ''
            /sgex;

            #   process fifth-level structure
            sub parse5a {
                my ($title, $txt) = @_;

                if (not exists $cache->{$title . ":" . $txt}) {
                    $html_navi .= "<li>" . mklink(0, $txt) . "<\/li>";
                    $cache->{$title . ":" . $txt} = 1;
                }
            }

            $html_spec .= "<div class=\"synopsis\">";
            $synopsis =~ s/[ \t]{2,}/ /sg;
            $synopsis = conv(1, $synopsis);
            $synopsis =~ s/(\[|\]|:|\s+=\s+)/<span class=\"meta\">$1<\/span>/sg;
            $synopsis =~ s/;/<\/div><div class="synopsis">/sg;
            $synopsis =~ s/\(\s+/(/sg;
            $synopsis =~ s/\s+\)/)/sg;
            $html_spec .= "$synopsis\n";
            $html_spec .= "</div>";

            #   parse fifth-level structure (part 2)
            my $txt = $body;
            $txt =~ s/
                (.+?\n)
                (?= \n | $ )
            /
                parse5b($1), ''
            /sgex;

            #   process fifth-level structure
            sub parse5b {
                my ($txt) = @_;

                if ($txt =~ m/^\s*\|\s/s) {
                    $txt =~ s/^\s*\|\s//mg;
                    $txt = conv(0, $txt);
                    $txt =~ s/\n+$//s;
                    $txt =~ s/^\s+//s;
                    $html_spec .= "<div class=\"example\">";
                    $html_spec .= "$txt\n";
                    $html_spec .= "</div>";
                }
                elsif ($txt =~ m/^\s*\.\s/s) {
                    $txt =~ s/([ \t]+)\.([ \t]+)(.+?\n(?:\1 \2.*?\n)*)/<li>$3<\/li>/sg;
                    $txt = conv(0, $txt);
                    $html_spec .= "<ul class=\"list\">";
                    $html_spec .= "$txt\n";
                    $html_spec .= "</ul>";
                }
                elsif ($txt =~ m/^\s*\+\s/s) {
                    $txt =~ s/^\s*\+\s//mg;
                    $txt = conv(0, $txt);
                    $txt =~ s/^(.*)$/<tr><td>$1<\/td><\/tr>/mg;<commit_msg>fix parsing of last block of API documentation<commit_after>##
##  ComponentJS -- Component System for JavaScript <http://componentjs.com>
##  Copyright (c) 2009-2013 <NAME> <http://engelschall.com>
##
##  This Source Code Form is subject to the terms of the Mozilla Public
##  License, v. 2.0. If a copy of the MPL was not distributed with this
##  file, You can obtain one at http://mozilla.org/MPL/2.0/.
##

use IO::File;

#   command-line arguments
my ($filename_txt, $filename_tmpl, $filename_html, $version) = @ARGV;

#   read textual description and template
my $fp = new IO::File "<$filename_txt"; $txt = ""; $txt .= $_ while (<$fp>); $fp->close();
$fp = new IO::File "<$filename_tmpl"; $tmpl = ""; $tmpl .= $_ while (<$fp>); $fp->close();

#   replace version information
$tmpl =~ s/0\.0\.0/$version/s;

#   start generating
my $html_spec = "";
my $html_navi = "";

#   duplication removal cache
my $cache = {};
my $defined = {};

#   remove comments
$txt =~ s/^#.*$//mg;

#   parse first-level structure
$txt =~ s/
    (?<=\n)([A-Z][^\n]+)\n  # headline
    ---+[ \t]*\n            # underlining
    (.+?\n)                 # body
    (?= [A-Z][^\n]+?\n      # following headline
        ---+[ \t]*\n        # corresponding underlining
    |   $             )     # or at end file
/
    parse2($1, $2), ''
/sgex;

#   process first-level structure
sub parse2 {
    my ($title, $body) = @_;

    #   generate body headline
    $html_spec .= "<h2>" . mklink(1, $title) . "</h2>\n";

    #   start navigation entry
    $html_navi .= "<h2>" . mklink(0, $title) . "</h2>\n";
    $html_navi .= "<ul>\n";

    #   parse second-level structure (part 1)
    $body =~ s/
        ^(.+?\n)           # intro paragraph
        (?=\n-[ \t]+\S+)   # start of first function
    /
        $html_spec .= "<div class=\"intro\">" . addpara(conv(0, $1)) . "<\/div>", ''
    /sex;
    sub addpara {
        my ($txt) = @_;
        $txt =~ s/(\n).[ \t]+([^\n]*(?:\n[ \t]+[^\n]+)*)/$1<li>$2<\/li>/sg;
        $txt =~ s/(\n\n)(<li>)/$1<ul>$2/sg;
        $txt =~ s/(<\/li>)(\n\n)/$1<\/ul>$2/sg;
        $txt =~ s/\n{2,}/<p\/>\n/sg;
        return $txt;
    }

    #   parse second-level structure (part 2)
    $body =~ s/
        (?<= \n)
        -[ \t]+(\S+.+?)    # function start
        (?= -[ \t]+\S+     # start of next function
        |   $         )    # or end of file
    /
        parse3($title, $1), ''
    /sgex;

    #   process third-level structure
    sub parse3 {
        my ($title, $body) = @_;

        $html_spec .= "<ul>\n";

        #   parse forth-level structure
        $body =~ s/
            ^
            (.+?)\n   #  synopsis
            \n        #  blank line
            (.+)      #  body
            $
        /
            parse4($title, $1, $2), ''
        /sex;

        #   process forth-level structure
        sub parse4 {
            my ($title, $synopsis, $body) = @_;

            $html_spec .= "<li>";

            #   parse fifth-level structure (part 1)
            my $txt = $synopsis;
            $txt =~ s/
                M<(.+?)>
            /
                parse5a($title, $1), ''
            /sgex;

            #   process fifth-level structure
            sub parse5a {
                my ($title, $txt) = @_;

                if (not exists $cache->{$title . ":" . $txt}) {
                    $html_navi .= "<li>" . mklink(0, $txt) . "<\/li>";
                    $cache->{$title . ":" . $txt} = 1;
                }
            }

            $html_spec .= "<div class=\"synopsis\">";
            $synopsis =~ s/[ \t]{2,}/ /sg;
            $synopsis = conv(1, $synopsis);
            $synopsis =~ s/(\[|\]|:|\s+=\s+)/<span class=\"meta\">$1<\/span>/sg;
            $synopsis =~ s/;/<\/div><div class="synopsis">/sg;
            $synopsis =~ s/\(\s+/(/sg;
            $synopsis =~ s/\s+\)/)/sg;
            $html_spec .= "$synopsis\n";
            $html_spec .= "</div>";

            #   parse fifth-level structure (part 2)
            my $txt = $body . "\n";
            $txt =~ s/
                (.+?\n)
                (?= \n | $ )
            /
                parse5b($1), ''
            /sgex;

            #   process fifth-level structure
            sub parse5b {
                my ($txt) = @_;

                if ($txt =~ m/^\s*\|\s/s) {
                    $txt =~ s/^\s*\|\s//mg;
                    $txt = conv(0, $txt);
                    $txt =~ s/\n+$//s;
                    $txt =~ s/^\s+//s;
                    $html_spec .= "<div class=\"example\">";
                    $html_spec .= "$txt\n";
                    $html_spec .= "</div>";
                }
                elsif ($txt =~ m/^\s*\.\s/s) {
                    $txt =~ s/([ \t]+)\.([ \t]+)(.+?\n(?:\1 \2.*?\n)*)/<li>$3<\/li>/sg;
                    $txt = conv(0, $txt);
                    $html_spec .= "<ul class=\"list\">";
                    $html_spec .= "$txt\n";
                    $html_spec .= "</ul>";
                }
                elsif ($txt =~ m/^\s*\+\s/s) {
                    $txt =~ s/^\s*\+\s//mg;
                    $txt = conv(0, $txt);
                    $txt =~ s/^(.*)$/<tr><td>$1<\/td><\/tr>/mg;
{'fraction_non_alphanumeric': 0.21904761904761905, 'fraction_numerical': 0.012244897959183673, 'mean_word_length': 1.2879057358826145, 'pattern_counts': {'":': 4, '<': 91, '<?xml version=': 0, '>': 91, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4171558', 'n_tokens_mistral': 3704, 'n_tokens_neox': 3505, 'n_words': 761}