<commit_before>xquery version "3.1";
(:~
 : BaseX full-text functions
 :
 : @see http://docs.basex.org/Full-Text_Module
 :)
module namespace ft = "http://basex.org/modules/ft";

declare namespace a = "http://reecedunn.co.uk/xquery/annotations";
declare namespace o = "http://reecedunn.co.uk/xquery/options";

declare option o:requires "basex/7.0"; (: NOTE: 7.0 is the earliest version definitions are available for. :)

declare %a:since("basex", "7.0") function ft:search($db as xs:string, $terms as item()*) as text()* external;
declare %a:since("basex", "7.2") function ft:search($db as xs:string, $terms as item()*, $options as map(*)?) as text()* external;
declare %a:since("basex", "7.8") function ft:contains($input as item()*, $terms as item()*) as xs:boolean external;
declare %a:since("basex", "7.8") function ft:contains($input as item()*, $terms as item()*, $options as map(*)?) as xs:boolean external;
declare %a:since("basex", "7.0") function ft:mark($nodes as node()*) as node()* external;
declare %a:since("basex", "7.0") function ft:mark($nodes as node()*, $name as xs:string) as node()* external;
declare %a:since("basex", "7.0") function ft:extract($nodes as node()*) as node()* external;
declare %a:since("basex", "7.0") function ft:extract($nodes as node()*, $name as xs:string) as node()* external;
declare %a:since("basex", "7.0") function ft:extract($nodes as node()*, $name as xs:string, $length as xs:integer) as node()* external;
declare %a:since("basex", "7.0") function ft:count($nodes as node()*) as xs:integer external;
declare %a:since("basex", "7.0") function ft:score($item as item()*) as xs:double* external;
declare %a:since("basex", "7.1") function ft:tokens($db as xs:string) as element(value)* external;
declare %a:since("basex", "7.1") function ft:tokens($db as xs:string, $prefix as xs:string) as element(value)* external;
declare %a:since("basex", "7.1") function ft:tokenize($string as xs:string?) as xs:string* external;
declare %a:since("basex", "7.1") function ft:tokenize($string as xs:string?, $options as map(*)?) as xs:string* external;
declare %a:since("basex", "8.0") function ft:normalize($string as xs:string?) as xs:string external;
declare %a:since("basex", "8.0") function ft:normalize($string as xs:string?, $options as map(*)?) as xs:string external;<commit_msg>BaseX: The ft:tokenize and ft:normalize string parameter allows an empty sequence in BaseX 9.1.<commit_after>xquery version "3.1";
(:~
 : BaseX full-text functions
 :
 : @see http://docs.basex.org/Full-Text_Module
 :)
module namespace ft = "http://basex.org/modules/ft";

declare namespace a = "http://reecedunn.co.uk/xquery/annotations";
declare namespace o = "http://reecedunn.co.uk/xquery/options";

declare option o:requires "basex/7.0"; (: NOTE: 7.0 is the earliest version definitions are available for. :)

declare type ft-string = (
    %a:since("basex", "7.0") %a:until("basex", "9.1") for xs:string |
    %a:since("basex", "9.1") for xs:string?
);

declare %a:since("basex", "7.0") function ft:search($db as xs:string, $terms as item()*) as text()* external;
declare %a:since("basex", "7.2") function ft:search($db as xs:string, $terms as item()*, $options as map(*)?) as text()* external;
declare %a:since("basex", "7.8") function ft:contains($input as item()*, $terms as item()*) as xs:boolean external;
declare %a:since("basex", "7.8") function ft:contains($input as item()*, $terms as item()*, $options as map(*)?) as xs:boolean external;
declare %a:since("basex", "7.0") function ft:mark($nodes as node()*) as node()* external;
declare %a:since("basex", "7.0") function ft:mark($nodes as node()*, $name as xs:string) as node()* external;
declare %a:since("basex", "7.0") function ft:extract($nodes as node()*) as node()* external;
declare %a:since("basex", "7.0") function ft:extract($nodes as node()*, $name as xs:string) as node()* external;
declare %a:since("basex", "7.0") function ft:extract($nodes as node()*, $name as xs:string, $length as xs:integer) as node()* external;
declare %a:since("basex", "7.0") function ft:count($nodes as node()*) as xs:integer external;
declare %a:since("basex", "7.0") function ft:score($item as item()*) as xs:double* external;
declare %a:since("basex", "7.1") function ft:tokens($db as xs:string) as element(value)* external;
declare %a:since("basex", "7.1") function ft:tokens($db as xs:string, $prefix as xs:string) as element(value)* external;
declare %a:since("basex", "7.1") function ft:tokenize($string as ft-string) as xs:string* external;
declare %a:since("basex", "7.1") function ft:tokenize($string as ft-string, $options as map(*)?) as xs:string* external;
declare %a:since("basex", "8.0") function ft:normalize($string as ft-string) as xs:string external;
declare %a:since("basex", "8.0") function ft:normalize($string as ft-string, $options as map(*)?) as xs:string external;
{'fraction_non_alphanumeric': 0.19740900678593462, 'fraction_numerical': 0.018095825622044006, 'mean_word_length': 6.36969696969697, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5707898', 'n_tokens_mistral': 1729, 'n_tokens_neox': 1631, 'n_words': 511}