<commit_before>:- module(
  term_ext,
  [
    ascii_id/1,            % -Id
    ascii_id/2,            % +Term, -Id
    compound_name/2,       % +Term, ?Name
    number_of_variables/2, % +Term, -NumberOfVariables
    replace_blobs/2,       % +Term1, -Term2
    write_fact/1,          % @Term
    write_term/1           % @Term
  ]
).

/** <module> Extended support for terms

Extends the support for terms in the SWI-Prolog standard library.

This module introduces the following types:

  - `options`: A dictionary with tag `options` that is used to
               represent options passed to predicates.

*/

:- use_module(library(apply)).
:- use_module(library(dict)).
:- use_module(library(error)).
:- use_module(library(uuid)).

:- use_module(library(hash_ext)).

:- multifile
    error:has_type/2.

error:has_type(options, Term) :-
  error:has_type(dict, Term),
  dict_key(Term, options).





%! ascii_id(-Id:atom) is det.

ascii_id(Id) :-
  uuid(Id0, [format(integer)]),
  atom_concat(n, Id0, Id).



%! ascii_id(+Term:term, -Id:atom) is det.
%
% Generates an Id for the given Prolog term.  The Id is guarantee to
% only contain ASCII letter and digits to ensure that it can be used
% by many programs.
%
% Prolog terms can contain many characters and can have arbitrary
% size.  For these reasons, this predicate generates the Id based on
% the MD5 hash of a serialization of the given Prolog term.
%
% MD5 hashes sometimes start with an ASCII digit, which is not
% supported in some languages/context (for example, the DOT language
% does not allow IDs to start with a digit).  For this reason, an
% arbitrary ASCII letter is prefixed at the beginning of the Id.
<commit_msg>Added a Prolog type declaration for maybe-values.<commit_after>:- module(
  term_ext,
  [
    ascii_id/1,            % -Id
    ascii_id/2,            % +Term, -Id
    compound_name/2,       % +Term, ?Name
    number_of_variables/2, % +Term, -NumberOfVariables
    replace_blobs/2,       % +Term1, -Term2
    write_fact/1,          % @Term
    write_term/1           % @Term
  ]
).

/** <module> Extended support for terms

Extends the support for terms in the SWI-Prolog standard library.

This module introduces the following types:

  - `options`: A dictionary with tag `options` that is used to
               represent options passed to predicates.

*/

:- use_module(library(apply)).
:- use_module(library(dict)).
:- use_module(library(error)).
:- use_module(library(uuid)).

:- use_module(library(hash_ext)).

:- multifile
    error:has_type/2.

error:has_type(maybe(Type), Term) :-
  ( error:has_type(Type, Term)
  ; error:has_type(var, Term)
  ).
error:has_type(options, Term) :-
  error:has_type(dict, Term),
  dict_key(Term, options).





%! ascii_id(-Id:atom) is det.

ascii_id(Id) :-
  uuid(Id0, [format(integer)]),
  atom_concat(n, Id0, Id).



%! ascii_id(+Term:term, -Id:atom) is det.
%
% Generates an Id for the given Prolog term.  The Id is guarantee to
% only contain ASCII letter and digits to ensure that it can be used
% by many programs.
%
% Prolog terms can contain many characters and can have arbitrary
% size.  For these reasons, this predicate generates the Id based on
% the MD5 hash of a serialization of the given Prolog term.
%
% MD5 hashes sometimes start with an ASCII digit, which is not
% supported in some languages/context (for example, the DOT language
% does not allow IDs to start with a digit).  For this reason, an
% arbitrary ASCII letter is prefixed at the beginning of the Id.

{'fraction_non_alphanumeric': 0.10778443113772455, 'fraction_numerical': 0.007984031936127744, 'mean_word_length': 3.1613285883748516, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '465195', 'n_tokens_mistral': 1166, 'n_tokens_neox': 1104, 'n_words': 388}