<commit_before>(:
 : Copyright 2006-2009 The FLWOR Foundation.
 :
 : Licensed under the Apache License, Version 2.0 (the "License");
 : you may not use this file except in compliance with the License.
 : You may obtain a copy of the License at
 :
 : http://www.apache.org/licenses/LICENSE-2.0
 :
 : Unless required by applicable law or agreed to in writing, software
 : distributed under the License is distributed on an "AS IS" BASIS,
 : WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 : See the License for the specific language governing permissions and
 : limitations under the License.
:)

(:~
 : This library module provides simple file system I/O operations.
 :
 : @author <NAME>
 :
 :)
module namespace file = "http://www.zorba-xquery.com/modules/file";

(:~
 : Copies a file given a source and a destination paths/URIs. The operation
 : fails if the source path/URI does not point to a file or the destination
 : path/URI is already used.
 :
 : @param $sourceFile The path/URI of the file to copy.
 : @param $destinationFile The detination <tt>path/URI</tt>.
 : @return true if the copy operation was successful.
 : @error An error is thrown if IO or Security problems occur.
 :)
declare sequential function file:copy(
  $sourceFile as xs:string,
  $destinationFile as xs:string
) as xs:boolean external;

(:~
 : Copies a file given a source and a destination paths/uris. The operation
 : fails if the source path/URI does not point to a file or the destination
 : path/URI is already used if the overwrite flag is se to false.
 :
 : @param $sourceFile The path/URI of the file to copy.
 : @param $destinationFile The destination path/URI.
 : @param $overwrite Flag to control if the operation should succeed if a the
 :        destination file already exists.
 : @return true if the copy operation was successful
 : @error An error is thrown if IO or Security problems occur.
 :)
declare sequential function file:copy(
  $sourceFile as xs:string,
  $destinationFile as xs:string,
  $overwrite as xs:boolean
) as xs:boolean external;

(:~
 : Tests if a path/URI is already used in the file system
 :
 : @param $fileOrDir The path/URI to test for existance.
 : @return true if the path/URI points to an existing file system item.
 : @error An error is thrown if IO or Security problems occur.
 :)
declare %nondeterministic function file:exists(
  $fileOrDir as xs:string
) as xs:boolean external;

(:~
 : Lists the file system items in a certain directory.
 : The order of the items in the result is not defined.
 : The "." and ".." items are not returned.
 : The file paths are relative to the provided <tt>$path</tt>.
 :
 : @param $path The path/URI to retrieve the children from.
 : @return The sequence of names of the direct children. 
 : @error An error is thrown if IO or Security problems occur.
 :)
declare %nondeterministic function file:files(
  $path as xs:string
) as xs:string* external;

(:~
 : Lists all files matching the given pattern in a given directory.
 : The order of the items in the result is not defined.
 : The "." and ".." items are not considered for the match.
 : The result of this function is equivalent to the following call:
 : <tt>file:files($path, $pattern, fn:false())</tt>
 : The file paths are relative to the provided <tt>$path</tt>.
 : 
 : @param $path The path/URI to retrieve the children from.
 : @param $pattern The file name condition to be checked.
 : @return A sequence of file names matching the pattern.
 :)
declare function file:files(
  $path as xs:string,
  $pattern as xs:string
) as xs:string* {
  file:files($path, $pattern, fn:false())
};

(:~
 : Lists all files matching the given pattern in a given directory.
 : The order of the items in the result is not defined.
 : The "." and ".." items are not considered for the match.
 : The file paths are relative to the provided <tt>$path</tt>.
 : 
 : @param $path The path/URI to retrieve the children from.
 : @param $pattern The file name condition to be checked.
 : @param $recursive A boolean flag indicating whether directories
 :        should be searched recursively.
 : @return A sequence of file names matching the pattern.
 :)
declare function file:files(
  $path as xs:string,
  $pattern as xs:string,
  $recursive as xs:boolean
) as xs:string* {
  for $f in file:files($path)
  let $full := fn:concat($path, "/", $f)
  let $is_directory := file:is-directory($full)
  let $result :=
    if ($recursive and $is_directory)
    then
      for $child in file:files($full, $pattern, $recursive)
      return fn:concat($f, file:path-separator(), $child)
    else
      if (fn:matches($f, $pattern))
      then $f
      else ()
  return $result
};

(:~
 : Tests if a path/URI points to a directory. On UNIX-based systems, the root
 : and the volumes roots are considered directories.
 :
 : @param $dir The path/URI to test.
 : @return true if the path/URI points to a directory.
 : @error An error is thrown if IO or Security problems occur.
 :)
declare %nondeterministic function file:is-directory(
  $dir as xs:string
) as xs:boolean external;

(:~
 : Tests if a path/URI points to a file.
 :
 : @param $dir The path/URI to test.
 : @return true if the path/URI points to a file.
 : @error An error is thrown if IO or Security problems occur.<commit_msg>Fixed SF#3102373. Recursive file:files with pattern was not working.<commit_after>(:
 : Copyright 2006-2009 The FLWOR Foundation.
 :
 : Licensed under the Apache License, Version 2.0 (the "License");
 : you may not use this file except in compliance with the License.
 : You may obtain a copy of the License at
 :
 : http://www.apache.org/licenses/LICENSE-2.0
 :
 : Unless required by applicable law or agreed to in writing, software
 : distributed under the License is distributed on an "AS IS" BASIS,
 : WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 : See the License for the specific language governing permissions and
 : limitations under the License.
:)

(:~
 : This library module provides simple file system I/O operations.
 :
 : @author <NAME>
 :
 :)
module namespace file = "http://www.zorba-xquery.com/modules/file";

(:~
 : Copies a file given a source and a destination paths/URIs. The operation
 : fails if the source path/URI does not point to a file or the destination
 : path/URI is already used.
 :
 : @param $sourceFile The path/URI of the file to copy.
 : @param $destinationFile The detination <tt>path/URI</tt>.
 : @return true if the copy operation was successful.
 : @error An error is thrown if IO or Security problems occur.
 :)
declare sequential function file:copy(
  $sourceFile as xs:string,
  $destinationFile as xs:string
) as xs:boolean external;

(:~
 : Copies a file given a source and a destination paths/uris. The operation
 : fails if the source path/URI does not point to a file or the destination
 : path/URI is already used if the overwrite flag is se to false.
 :
 : @param $sourceFile The path/URI of the file to copy.
 : @param $destinationFile The destination path/URI.
 : @param $overwrite Flag to control if the operation should succeed if a the
 :        destination file already exists.
 : @return true if the copy operation was successful
 : @error An error is thrown if IO or Security problems occur.
 :)
declare sequential function file:copy(
  $sourceFile as xs:string,
  $destinationFile as xs:string,
  $overwrite as xs:boolean
) as xs:boolean external;

(:~
 : Tests if a path/URI is already used in the file system
 :
 : @param $fileOrDir The path/URI to test for existance.
 : @return true if the path/URI points to an existing file system item.
 : @error An error is thrown if IO or Security problems occur.
 :)
declare %nondeterministic function file:exists(
  $fileOrDir as xs:string
) as xs:boolean external;

(:~
 : Lists the file system items in a certain directory.
 : The order of the items in the result is not defined.
 : The "." and ".." items are not returned.
 : The file paths are relative to the provided <tt>$path</tt>.
 :
 : @param $path The path/URI to retrieve the children from.
 : @return The sequence of names of the direct children. 
 : @error An error is thrown if IO or Security problems occur.
 :)
declare %nondeterministic function file:files(
  $path as xs:string
) as xs:string* external;

(:~
 : Lists all files matching the given pattern in a given directory.
 : The order of the items in the result is not defined.
 : The "." and ".." items are not considered for the match.
 : The result of this function is equivalent to the following call:
 : <tt>file:files($path, $pattern, fn:false())</tt>
 : The file paths are relative to the provided <tt>$path</tt>.
 : 
 : @param $path The path/URI to retrieve the children from.
 : @param $pattern The file name condition to be checked.
 : @return A sequence of file names matching the pattern.
 :)
declare function file:files(
  $path as xs:string,
  $pattern as xs:string
) as xs:string* {
  file:files($path, $pattern, fn:false())
};

(:~
 : Lists all files matching the given pattern in a given directory.
 : The order of the items in the result is not defined.
 : The "." and ".." items are not considered for the match.
 : The file paths are relative to the provided <tt>$path</tt>.
 : 
 : @param $path The path/URI to retrieve the children from.
 : @param $pattern The file name condition to be checked.
 : @param $recursive A boolean flag indicating whether directories
 :        should be searched recursively.
 : @return A sequence of file names matching the pattern.
 :)
declare function file:files(
  $path as xs:string,
  $pattern as xs:string,
  $recursive as xs:boolean
) as xs:string* {
  for $f in file:files($path)
  let $full := fn:concat($path, file:path-separator(), $f)
  let $is_directory := file:is-directory($full)
  let $result :=
    (
      if (fn:matches($f, $pattern))
      then $f
      else ()
      ,
      if ($recursive and $is_directory)
      then
        for $child in file:files($full, $pattern, $recursive)
        return fn:concat($f, file:path-separator(), $child)
      else ()
  )
  return $result
};

(:~
 : Tests if a path/URI points to a directory. On UNIX-based systems, the root
 : and the volumes roots are considered directories.
 :
 : @param $dir The path/URI to test.
 : @return true if the path/URI points to a directory.
 : @error An error is thrown if IO or Security problems occur.
 :)
declare %nondeterministic function file:is-directory(
  $dir as xs:string
) as xs:boolean external;

(:~
 : Tests if a path/URI points to a file.
 :
 : @param $dir The path/URI to test.
 : @return true if the path/URI points to a file.
 : @error An error is thrown if IO or Security problems occur.
{'fraction_non_alphanumeric': 0.09383177570093458, 'fraction_numerical': 0.002897196261682243, 'mean_word_length': 3.6105127100387766, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2640799', 'n_tokens_mistral': 3106, 'n_tokens_neox': 2976, 'n_words': 1448}