<commit_before># $Id$
# Copyright (C) 2001-2007, The Perl Foundation.

package Scheme::Parser;

# pragmata
use strict;
use warnings;
use 5.008;

our $VERSION   = '0.01';

use Data::Dumper;

# walk over the tokens
sub _build_tree {
    my ( $tokenizer ) = @_;

    my $token = $tokenizer->();
   
    return unless $token;

    return if $token->[1] eq ')';

    if ( $token->[1] eq '(' ) {
        my @children;
        while ( my $child = _build_tree( $tokenizer ) ) {
            push @children, $child;
        }

        if ( ! @children ) {
            # special case: empty list
            return { value => undef };
        }
        else {
            return { children => \@children };
        }
    } 

    my %function = (
	    q{'}      => 'quote',
	    q{`}      => 'quasiquote',
	    q{,}      => 'unquote',
            q{,@}     => 'unquote-splicing',
    );
    if ( exists $function{$token->[1]}  ) {
        my $tree = { children => [ { value => $function{$token->[1]} 
                                   }
                                 ]
                   };
        while ( my $child = _build_tree( $tokenizer ) ) {
             push @{ $tree->{children} }, $child;
        }

	return $tree;
    }

    # the atomic case
    return { value => $token->[1] };
}


sub parse {
    my $tokenizer = shift;

    my @trees;
    while ( my $tree = _build_tree( $tokenizer ) ) {
        push @trees, $tree;
    }

    return undef unless @trees;

    return $trees[0] if scalar(@trees) == 1;

    # Implicit begin at toplevel
    return { children => [ { value => 'begin' },
                           @trees
                         ]
           };
}
<commit_msg>[Scheme] Use wantarray() in _build_tree(). First time I ever used wantarray().<commit_after># $Id$
# Copyright (C) 2001-2007, The Perl Foundation.

package Scheme::Parser;

# pragmata
use strict;
use warnings;
use 5.008;

our $VERSION   = '0.01';

use Data::Dumper;

# walk over the tokens
sub _build_tree {
    my ( $tokenizer ) = @_;

    if ( wantarray() ) {
        my @trees;
        while ( my $tree = _build_tree( $tokenizer ) ) {
            push @trees, $tree;
        }

        return @trees;
    }

    my $token = $tokenizer->();
   
    return unless $token;

    return if $token->[1] eq ')';

    if ( $token->[1] eq '(' ) {
        my @children = _build_tree( $tokenizer );

        if ( ! @children ) {
            # special case: empty list
            return { value => undef };
        }
        else {
            return { children => \@children };
        }
    } 

    my %function = (
	    q{'}      => 'quote',
	    q{`}      => 'quasiquote',
	    q{,}      => 'unquote',
            q{,@}     => 'unquote-splicing',
    );
    if ( exists $function{$token->[1]}  ) {
	return { children => [ { value => $function{$token->[1]} 
                               },
                                _build_tree( $tokenizer )
                             ]
               };

    }

    # the atomic case
    return { value => $token->[1] };
}


sub parse {
    my $tokenizer = shift;

    my @trees = _build_tree( $tokenizer );

    return undef unless @trees;

    return $trees[0] if scalar(@trees) == 1;

    # Implicit begin at toplevel
    return { children => [ { value => 'begin' },
                           @trees
                         ]
           };
}

{'fraction_non_alphanumeric': 0.15176715176715178, 'fraction_numerical': 0.013068013068013067, 'mean_word_length': 1.465592972181552, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6422823', 'n_tokens_mistral': 1131, 'n_tokens_neox': 1038, 'n_words': 269}