<commit_before>:- module(
  gnu_sort,
  [
    gnu_sort/2,  % +File, +Opts
    sort_file/1, % +File
    sort_file/2  % +File, +Opts
  ]
).

/** <module> GNU sort

Support for calling GNU sort from within Prolog.

  * gnu_sort/2 is the low-level version, performing a one-to-one
    mapping from Prolog options to CLI flags.

  * sort_file/2 is the high-level version that simplifies the GNU Sort
    API.

@author <NAME>
@version 2015/08-2015/10, 2016/01
*/

:- use_module(library(cli_ext)).
:- use_module(library(debug)).
:- use_module(library(error)).
:- use_module(library(option)).
:- use_module(library(os/process_ext)).
:- use_module(library(os/thread_ext)).
:- use_module(library(typecheck)).





%! gnu_sort(+File, +Opts) is det.
%
% The following options are supported:
%
%   * buffer_size(+nonneg)
%
%     The size of the buffer in kilobytes.
%
%   * duplicates(+boolean)
%
%     Whether duplicates are allowed in the result.  Default is
%     `true`.
%
%   * numeric(+boolean)
%
%     Whether numberic sort is performed.  Default is `false`.
%
%   * output(+atom)
%
%     The name of the output file.
%
%   * temporary_directory(+atom)
%
%     The directory that is used for temporary sort results.  By
%     default this is the directory of the source file.
%
%   * threads(+positive_integer)
%
%     The number of threads that is used.  Default is the number of
%     available processors, but not larger than 8.  Larger numbers
%     have diminishing returns.
%
%   * utf8(+boolean)
%
%     Whether the environment is set to UTF-8 encoding.  Default is
%     `true`.

gnu_sort(File, Opts) :-
  must_be_file(read, File),
  % The UTF-8 encoding option is handled by an environment variable.
  (option(utf8(true), Opts, true) -> Env = [] ; Env = ['LC_ALL'='C']),
  gnu_sort_args(Opts, Args),
  run_process(sort, [file(File)|Args], [env(Env),program('GNU sort')]).


gnu_sort_args([], []).
gnu_sort_args([buffer_size(Size)|T1], [Arg|T2]) :- !,
  long_flag('buffer-size', Size, Arg),
  gnu_sort_args(T1, T2).
gnu_sort_args([duplicates(false)|T1], ['--unique'|T2]) :- !,
  gnu_sort_args(T1, T2).
gnu_sort_args([numeric(Numeric)|T1], [Arg|T2]) :- !,
  must_be(boolean, Numeric),
  long_flag('numeric-sort', Arg),
  gnu_sort_args(T1, T2).
gnu_sort_args([output(File)|T1], [Arg|T2]) :- !,
  must_be_file(write, File),
  long_flag(output, File, Arg),
  gnu_sort_args(T1, T2).
gnu_sort_args([threads(Threads)|T1], [Arg|T2]) :- !,
  long_flag(parallel, Threads, Arg),
  gnu_sort_args(T1, T2).<commit_msg>minor<commit_after>:- module(
  gnu_sort,
  [
    gnu_sort/2,  % +File, +Opts
    sort_file/1, % +File
    sort_file/2  % +File, +Opts
  ]
).

/** <module> GNU sort

Support for calling GNU sort from within Prolog.

  * gnu_sort/2 is the low-level version, performing a one-to-one
    mapping from Prolog options to CLI flags.

  * sort_file/2 is the high-level version that simplifies the GNU Sort
    API.

@author Wouter Beek
@version 2015/08-2015/10, 2016/01
*/

:- use_module(library(cli_ext)).
:- use_module(library(debug)).
:- use_module(library(error)).
:- use_module(library(option)).
:- use_module(library(os/thread_ext)).
:- use_module(library(process)).
:- use_module(library(typecheck)).





%! gnu_sort(+File, +Opts) is det.
%
% The following options are supported:
%
%   * buffer_size(+nonneg)
%
%     The size of the buffer in kilobytes.
%
%   * duplicates(+boolean)
%
%     Whether duplicates are allowed in the result.  Default is
%     `true`.
%
%   * numeric(+boolean)
%
%     Whether numberic sort is performed.  Default is `false`.
%
%   * output(+atom)
%
%     The name of the output file.
%
%   * temporary_directory(+atom)
%
%     The directory that is used for temporary sort results.  By
%     default this is the directory of the source file.
%
%   * threads(+positive_integer)
%
%     The number of threads that is used.  Default is the number of
%     available processors, but not larger than 8.  Larger numbers
%     have diminishing returns.
%
%   * utf8(+boolean)
%
%     Whether the environment is set to UTF-8 encoding.  Default is
%     `true`.

gnu_sort(File, Opts1) :-
  must_be_file(read, File),
  % The UTF-8 encoding option is handled by an environment variable.
  (option(utf8(true), Opts1, true) -> Env = [] ; Env = ['LC_ALL'='C']),
  merge_options(Opts1, [output(File)], Opts2),
  gnu_sort_args(Opts2, Args),
  format(user_output, "<<<~w~n", [File]),
  process_create(path(sort), [file(File)|Args], [env(Env)]),
  format(user_output, "~w>>>~n", [File]).


gnu_sort_args([], []).
gnu_sort_args([buffer_size(Size)|T1], [Arg|T2]) :- !,
  long_flag('buffer-size', Size, Arg),
  gnu_sort_args(T1, T2).
gnu_sort_args([duplicates(false)|T1], ['--unique'|T2]) :- !,
  gnu_sort_args(T1, T2).
gnu_sort_args([numeric(Numeric)|T1], [Arg|T2]) :- !,
  must_be(boolean, Numeric),
  long_flag('numeric-sort', Arg),
  gnu_sort_args(T1, T2).
gnu_sort_args([output(File)|T1], [Arg|T2]) :- !,
  must_be_file(write, File),
  long_flag(output, File, Arg),
  gnu_sort_args(T1, T2).
gnu_sort_args([threads(Threads)|T1], [Arg|T2]) :- !,
  long_flag(parallel, Threads, Arg),
  gnu_sort_args(T1, T2).
{'fraction_non_alphanumeric': 0.16058536585365854, 'fraction_numerical': 0.019707317073170732, 'mean_word_length': 3.886558627264061, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3910126', 'n_tokens_mistral': 1976, 'n_tokens_neox': 1870, 'n_words': 454}