<commit_before>(:~
    KWIC module: formats query results to display <em>keywords in context</em> (KWIC). A configurable
    amount of text is displayed to the left and right of a matching keyword (or phrase).
    
    The module works with all indexes that support match highlighting (matches are tagged
    with an &lt;exist:match&gt; element or exist:matches attribute). This includes the old 
    full text index, the new Lucene-based full text index, as well as the NGram index.
    
    The <b>kwic:summarize()</b> function represents the main entry point into the module.
    To have more control over the text extraction context, you can also call 
    <b>kwic:get-summary()</b> instead. For example, the following snippet will only print the
    first match within a given set of context nodes ($ancestor):
    
    <pre>
    let $matches := kwic:get-matches($hit)<br/>
    for $ancestor in $matches/ancestor::para | $matches/ancestor::title | $matches/ancestor::td<br/>
    return<br/>
        kwic:get-summary($ancestor, ($ancestor//exist:match, $ancestor//*[@exist:matches])[1], $config)
    </pre>
    
    The configuration element used by some functions has the following structure:
    
    <config
        width="character width"
        table="yes|no"
        link="URL to which the match is linked"
        whitespace-text-nodes="collapse|drop|leave"
        debug-text-nodes="yes|no"/>
:)
xquery version "3.0";

module namespace kwic = "http://exist-db.org/xquery/kwic";

import module namespace util = "http://exist-db.org/xquery/util";

declare namespace exist = "http://exist.sourceforge.net/NS/exist";

declare variable $kwic:MODE_BEFORE := "before";
declare variable $kwic:MODE_AFTER := "after";

declare %private variable $kwic:CHARS_SUMMARY := 120;
declare %private variable $kwic:CHARS_KWIC := 40;


(:~
	Like fn:substring, but takes a node argument. If the node is an element,
	a new element is created with the same node-name as the old one and the
	shortened text content.
:)
declare
    %private
function kwic:substring($node as item(), $start as xs:integer, $count as xs:integer) as node() {
	let $str := substring($node, $start, $count)
	return
		if ($node instance of element()) then
			element { node-name($node) } { $str }
		else
			text { $str }
};

declare
    %private
function kwic:display-text($text as text()?) as node()? {
    if ($text/parent::exist:match) then
    	<span class="hi">{$text}</span>
    else
        $text
};

declare
    %private
function kwic:callback($callback as (function(text(), xs:string) as text()?)?, $node as text(), $mode as xs:string) as text()? {
    if (exists($callback)) then
        $callback($node, $mode)
    else
        $node
};

(:~
	Generate the left-hand context of the match. Returns a sequence of nodes
	and strings, whose total string length is less than or equal to $max characters.

	Note: this function calls itself recursively until $nodes is empty or
	the returned sequence has the desired total string length.
:)
declare
    %private
function kwic:truncate-previous($root as node(), $node as node()?, $truncated as item()*, 
	   $max as xs:integer, $chars as xs:integer, $callback as (function(text(), xs:string) as text()?)?) {
	if ($node) then
		let $next := $node/preceding::text()[1] intersect $root//text()
		return
			if (empty($next)) then
				$truncated
			else if ($chars + string-length($next) gt $max) then
			    let $str := kwic:callback($callback, $next, $kwic:MODE_BEFORE)
			    return
    			    if (exists($str)) then
        				let $remaining := $max - $chars
        				return (
        				    text { "..." },
        				    kwic:substring($str, string-length($str) - $remaining + 1, $remaining),
        				    $truncated
        			    )
        			else
        			    kwic:truncate-previous($root, $next, $truncated, $max, $chars, $callback)
			else
				kwic:truncate-previous($root, $next, 
				    (kwic:callback($callback, $next, $kwic:MODE_BEFORE), $truncated),
					$max, $chars + string-length($next), $callback)
	else
		$truncated
};

(:~
	Generate the right-hand context of the match. Returns a sequence of nodes
	and strings, whose total string length is less than or equal to $max characters.
	
	Note: this function calls itself recursively until $nodes is empty or
	the returned sequence has the desired total string length.
:)
declare
    %private
function kwic:truncate-following($root as node(), $node as node()?, $truncated as item()*, 
	$max as xs:integer, $chars as xs:integer, $callback as (function(text(), xs:string) as text()?)?) {
	if ($node) then
		let $next := $node/following::text()[1] intersect $root//text()
		return
			if (empty($next)) then
				$truncated
			else if ($chars + string-length($next) gt $max) then
			    let $str := kwic:callback($callback, $next, $kwic:MODE_AFTER)
			    return
    			    if (exists($str)) then
        				let $remaining := $max - $chars
        				return (
                            $truncated, kwic:substring($str, 1, $remaining), text { "..." }
        			    )
        			else
        			    kwic:truncate-following($root, $next, $truncated, $max, $chars, $callback)
			else
				kwic:truncate-following($root, $next, 
				    ($truncated, kwic:callback($callback, $next, $kwic:MODE_AFTER)),
					$max, $chars + string-length($next), $callback)
	else
		$truncated<commit_msg>[ignore] minor tidy-up<commit_after>(:~
    KWIC module: formats query results to display <em>keywords in context</em> (KWIC). A configurable
    amount of text is displayed to the left and right of a matching keyword (or phrase).
    
    The module works with all indexes that support match highlighting (matches are tagged
    with an &lt;exist:match&gt; element or exist:matches attribute). This includes the old 
    full text index, the new Lucene-based full text index, as well as the NGram index.
    
    The <b>kwic:summarize()</b> function represents the main entry point into the module.
    To have more control over the text extraction context, you can also call 
    <b>kwic:get-summary()</b> instead. For example, the following snippet will only print the
    first match within a given set of context nodes ($ancestor):
    
    <pre>
    let $matches := kwic:get-matches($hit)<br/>
    for $ancestor in $matches/ancestor::para | $matches/ancestor::title | $matches/ancestor::td<br/>
    return<br/>
        kwic:get-summary($ancestor, ($ancestor//exist:match, $ancestor//*[@exist:matches])[1], $config)
    </pre>
    
    The configuration element used by some functions has the following structure:
    
    <config
        width="character width"
        table="yes|no"
        link="URL to which the match is linked"
        whitespace-text-nodes="collapse|drop|leave"
        debug-text-nodes="yes|no"/>
:)
xquery version "3.0";

module namespace kwic = "http://exist-db.org/xquery/kwic";

import module namespace util = "http://exist-db.org/xquery/util";

declare namespace exist = "http://exist.sourceforge.net/NS/exist";

declare variable $kwic:MODE_BEFORE := "before";
declare variable $kwic:MODE_AFTER := "after";

declare %private variable $kwic:CHARS_SUMMARY := 120;
declare %private variable $kwic:CHARS_KWIC := 40;


(:~
	Like fn:substring, but takes a node argument. If the node is an element,
	a new element is created with the same node-name as the old one and the
	shortened text content.
:)
declare
    %private
function kwic:substring($node as text(), $start as xs:integer, $count as xs:integer) as node() {
	let $str := substring($node, $start, $count)
	return
		if ($node instance of element()) then
			element { node-name($node) } { $str }
		else
			text { $str }
};

declare
    %private
function kwic:display-text($text as text()?) as node()? {
    if ($text/parent::exist:match) then
    	<span class="hi">{$text}</span>
    else
        $text
};

declare
    %private
function kwic:callback($callback as (function(text(), xs:string) as text()?)?, $node as text(), $mode as xs:string) as text()? {
    if (exists($callback)) then
        $callback($node, $mode)
    else
        $node
};

(:~
	Generate the left-hand context of the match. Returns a sequence of nodes
	and strings, whose total string length is less than or equal to $max characters.

	Note: this function calls itself recursively until $nodes is empty or
	the returned sequence has the desired total string length.
:)
declare
    %private
function kwic:truncate-previous($root as node(), $node as node()?, $truncated as item()*, 
	   $max as xs:integer, $chars as xs:integer, $callback as (function(text(), xs:string) as text()?)?) {
	if ($node) then
		let $next := $node/preceding::text()[1] intersect $root//text()
		return
			if (empty($next)) then
				$truncated
			else if ($chars + string-length($next) gt $max) then
			    let $text-node := kwic:callback($callback, $next, $kwic:MODE_BEFORE)
			    return
    			    if (exists($text-node)) then
        				let $remaining := $max - $chars
        				return (
        				    text { "..." },
        				    kwic:substring($text-node, string-length($text-node) - $remaining + 1, $remaining),
        				    $truncated
        			    )
        			else
        			    kwic:truncate-previous($root, $next, $truncated, $max, $chars, $callback)
			else
				kwic:truncate-previous($root, $next, 
				    (kwic:callback($callback, $next, $kwic:MODE_BEFORE), $truncated),
					$max, $chars + string-length($next), $callback)
	else
		$truncated
};

(:~
	Generate the right-hand context of the match. Returns a sequence of nodes
	and strings, whose total string length is less than or equal to $max characters.
	
	Note: this function calls itself recursively until $nodes is empty or
	the returned sequence has the desired total string length.
:)
declare
    %private
function kwic:truncate-following($root as node(), $node as node()?, $truncated as item()*, 
	$max as xs:integer, $chars as xs:integer, $callback as (function(text(), xs:string) as text()?)?) {
	if ($node) then
		let $next := $node/following::text()[1] intersect $root//text()
		return
			if (empty($next)) then
				$truncated
			else if ($chars + string-length($next) gt $max) then
			    let $text-node := kwic:callback($callback, $next, $kwic:MODE_AFTER)
			    return
    			    if (exists($text-node)) then
        				let $remaining := $max - $chars
        				return (
                            $truncated,
                            kwic:substring($text-node, 1, $remaining),
                            text { "..." }
        			    )
        			else
        			    kwic:truncate-following($root, $next, $truncated, $max, $chars, $callback)
			else
				kwic:truncate-following($root, $next, 
				    ($truncated, kwic:callback($callback, $next, $kwic:MODE_AFTER)),
					$max, $chars + string-length($next), $callback)
	else
		$truncated
{'fraction_non_alphanumeric': 0.13059116480678779, 'fraction_numerical': 0.0022134095729964033, 'mean_word_length': 3.1373521556657766, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5505297', 'n_tokens_mistral': 3640, 'n_tokens_neox': 3293, 'n_words': 1101}