<commit_before>! Copyright (C) 2008 <NAME>.
! See http://factorcode.org/license.txt for BSD license.
USING: arrays help.markup help.syntax io.streams.string
sequences strings math suffix-arrays.private ;
IN: suffix-arrays

HELP: >suffix-array
{ $values
     { "seq" sequence }
     { "array" array } }
{ $description "Creates a suffix array from the input sequence.  Suffix arrays are arrays of slices." } ;

HELP: SA{
{ $description "Creates a new literal suffix array at parse-time." } ;

HELP: suffixes
{ $values
     { "string" string }
     { "suffixes-seq" "a sequence of slices" } }
{ $description "Returns a sequence of tail slices of the input string." } ;

HELP: from-to
{ $values
     { "index" integer } { "begin" sequence } { "suffix-array" "a suffix-array" }
     { "from/f" "an integer or f" } { "to/f" "an integer or f" } }
{ $description "Finds the bounds of the suffix array that match the input sequence. A return value of " { $link f } " means that the endpoint is included." }
{ $notes "Slices are [m,n) and we want (m,n) so we increment." } ;

HELP: query
{ $values
     { "begin" sequence } { "suffix-array" "a suffix-array" }
     { "matches" array } }
{ $description "Returns a sequence of sequences from the suffix-array that contain the input sequence. An empty array is returned when there are no matches." } ;

ARTICLE: "suffix-arrays" "Suffix arrays"
"The " { $vocab-link "suffix-arrays" } " vocabulary implements the suffix array data structure for efficient lookup of subsequences. This suffix array implementation is a sorted array of suffixes. Querying it for matches uses binary search for efficiency." $nl

"Creating new suffix arrays:"
{ $subsections >suffix-array }
"Literal suffix arrays:"
{ $subsections POSTPONE: SA{ }
"Querying suffix arrays:"
{ $subsections query } ;

ABOUT: "suffix-arrays"
<commit_msg>suffix-arrays: remove docs of removed private word.<commit_after>! Copyright (C) 2008 <NAME>.
! See http://factorcode.org/license.txt for BSD license.
USING: arrays help.markup help.syntax io.streams.string
sequences strings math suffix-arrays.private ;
IN: suffix-arrays

HELP: >suffix-array
{ $values
     { "seq" sequence }
     { "array" array } }
{ $description "Creates a suffix array from the input sequence.  Suffix arrays are arrays of slices." } ;

HELP: SA{
{ $description "Creates a new literal suffix array at parse-time." } ;

HELP: suffixes
{ $values
     { "string" string }
     { "suffixes-seq" "a sequence of slices" } }
{ $description "Returns a sequence of tail slices of the input string." } ;

HELP: query
{ $values
     { "begin" sequence } { "suffix-array" "a suffix-array" }
     { "matches" array } }
{ $description "Returns a sequence of sequences from the suffix-array that contain the input sequence. An empty array is returned when there are no matches." } ;

ARTICLE: "suffix-arrays" "Suffix arrays"
"The " { $vocab-link "suffix-arrays" } " vocabulary implements the suffix array data structure for efficient lookup of subsequences. This suffix array implementation is a sorted array of suffixes. Querying it for matches uses binary search for efficiency." $nl

"Creating new suffix arrays:"
{ $subsections >suffix-array }
"Literal suffix arrays:"
{ $subsections POSTPONE: SA{ }
"Querying suffix arrays:"
{ $subsections query } ;

ABOUT: "suffix-arrays"

{'fraction_non_alphanumeric': 0.11371137113711371, 'fraction_numerical': 0.0024002400240024004, 'mean_word_length': 3.9539375928677565, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2880847', 'n_tokens_mistral': 1022, 'n_tokens_neox': 969, 'n_words': 396}