<commit_before>class Text::Markov;

has %!graph;
has Int $.dimensions is rw = 1;

method feed ( *@o ) {

    # convert Array of objects into multidimensional Hash of predecessors
    # that ends with KeyBag containing successors with occurrence weights
    for ^@o.elems -> $i {
        
        # pointer starts at the beginning of predecessors Hash
        # and will eventually reach successors KeyBag
        my $p := %!graph;
        
        # create Hash dimensions of predecessors
        for ( ^$.dimensions ).reverse -> $j {
            
            # move pointer to next Hash dimension,
            # use empty string if predecessor is not available
            $p := $p{ ( $i - $j < 1 )  ?? '' !! @o[ $i - $j - 1 ] };
        }
        
        # successors KeyBag may not be created yet
        $p //= KeyBag.new;
        
        # increase occurrence weight for current successor
        $p{ @o[ $i ] }++;
    }
    
    return True;
}

method read ( Int $l? ) {

    # output Array of objects
    my @o;
    
    # find successors KeyBag in Hash
    loop {
        
        # pointer starts at the beginning of predecessors Hash
        # and will eventually reach successors KeyBag
        my $p := %!graph;
        
        # move through Hash dimensions using predecessors from output
        for ( ^$.dimensions ).reverse -> $i {
            
            # move pointer to next Hash dimension,
            # use empty string if predecessor is not available
            $p := $p{ ( @o.elems - $i > 0 ) ?? @o[ * - $i - 1 ] !! '' };
        }
        
        # no successors are available
        last unless $p ~~ KeyBag;
        
        # choose successor based on occurrence weights
        @o.push: $p.roll( );
        
        # finish if desired length is reached
        last if defined $l and @o.elems ~~ $l;
    }
    
    return @o;
}
<commit_msg>Add default length, infinite is too dangerous<commit_after>class Text::Markov;

has %!graph;
has Int $.dimensions is rw = 1;

method feed ( *@o ) {

    # convert Array of objects into multidimensional Hash of predecessors
    # that ends with KeyBag containing successors with occurrence weights
    for ^@o.elems -> $i {
        
        # pointer starts at the beginning of predecessors Hash
        # and will eventually reach successors KeyBag
        my $p := %!graph;
        
        # create Hash dimensions of predecessors
        for ( ^$.dimensions ).reverse -> $j {
            
            # move pointer to next Hash dimension,
            # use empty string if predecessor is not available
            $p := $p{ ( $i - $j < 1 )  ?? '' !! @o[ $i - $j - 1 ] };
        }
        
        # successors KeyBag may not be created yet
        $p //= KeyBag.new;
        
        # increase occurrence weight for current successor
        $p{ @o[ $i ] }++;
    }
    
    return True;
}

method read ( Int $l = 1024 ) {

    # output Array of objects
    my @o;
    
    # find successors KeyBag in Hash
    loop {
        
        # pointer starts at the beginning of predecessors Hash
        # and will eventually reach successors KeyBag
        my $p := %!graph;
        
        # move through Hash dimensions using predecessors from output
        for ( ^$.dimensions ).reverse -> $i {
            
            # move pointer to next Hash dimension,
            # use empty string if predecessor is not available
            $p := $p{ ( @o.elems - $i > 0 ) ?? @o[ * - $i - 1 ] !! '' };
        }
        
        # no successors are available
        last unless $p ~~ KeyBag;
        
        # choose successor based on occurrence weights
        @o.push: $p.roll( );
        
        # finish if desired length is reached
        last if defined $l and @o.elems ~~ $l;
    }
    
    return @o;
}

{'fraction_non_alphanumeric': 0.09678270042194093, 'fraction_numerical': 0.003691983122362869, 'mean_word_length': 1.6285516285516286, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4695012', 'n_tokens_mistral': 1071, 'n_tokens_neox': 1028, 'n_words': 409}