<commit_before>:- module(
  xml_ext,
  [
    call_on_xml/3,     % +UriSpec, +RecordName, :Goal_1
    call_on_xml/4,     % +UriSpec, +RecordName, :Goal_1, +Options
    html_download/2,   % +UriSpec, -Dom
    html_download/3,   % +UriSpec, -Dom, +Options
    html_insert_dom//1 % +Dom
  ]
).

/** <module> XML extensions

@author <NAME>
@version 2016/06-2017/08
*/

:- use_module(library(atom_ext)).
:- use_module(library(c14n2)).
:- use_module(library(http/html_write)).
:- use_module(library(option)).
:- use_module(library(sgml)).
:- use_module(library(uri/uri_ext)).

:- meta_predicate
    call_on_xml(+, +, 1),
    call_on_xml(+, +, 1, +),
    call_on_xml_stream(+, 1, +, +, -).





%! call_on_xml(+UriSpec:term, +RecordName:atom, :Goal_1) is det.
%! call_on_xml(+UriSpec:term, +RecordName:atom, :Goal_1,
%!             +Options:list(compound)) is det.
%
% Call Goal_1 on an XML stream, where the argument supplied to Goal_1
% is a subtree that starts with an element called RecordName.

call_on_xml(UriSpec, RecordName, Goal_1) :-
  call_on_xml(UriSpec, RecordName, Goal_1, []).


call_on_xml(UriSpec, RecordName, Goal_1, Options) :-
  call_on_uri(UriSpec, call_on_xml_stream(RecordName, Goal_1), Options).

call_on_xml_stream(RecordName, Goal_1, In, Metadata, Metadata) :-
  b_setval(xml_stream_goal, Goal_1),
  b_setval(xml_stream_record_name, RecordName),
  setup_call_cleanup(
    new_sgml_parser(Parser, []),
    (
      set_sgml_parser(Parser, space(remove)),
      sgml_parse(Parser, [call(begin,on_begin0),source(In)])
    ),
    free_sgml_parser(Parser)
  ).

on_begin0(Elem, Attr, Parser) :-
  b_getval(xml_stream_goal, Goal_1),
  b_getval(xml_stream_record_name, Elem), !,
  sgml_parse(Parser, [document(Dom1),parse(content)]),
  xml_clean_dom(Dom1, Dom2),
  call(Goal_1, [element(Elem,Attr,Dom2)]).

xml_clean_dom([element(Elem,Attr,Dom1)|T1], [element(Elem,Attr,Dom2)|T2]) :- !,
  xml_clean_dom(Dom1, Dom2),
  xml_clean_dom(T1, T2).
xml_clean_dom([H|T1], T2) :-
  is_empty_atom(H), !,
  xml_clean_dom(T1, T2).
xml_clean_dom([H1|T1], [H2|T2]) :- !,
  atom_strip(H1, H2),
  xml_clean_dom(T1, T2).
xml_clean_dom([], []).



%! html_download(+UriSpec:term, -Dom:list(compound)) is det.
%! html_download(+UriSpec:term, -Dom:list(compound),
%!               +Options:list(compound)) is det.

html_download(UriSpec, Dom) :-
  html_download(UriSpec, Dom, []).


html_download(UriSpec, Dom, Options) :-
  call_on_uri(UriSpec, html_download_(Dom, Options), Options).

html_download_(Dom2, Options1, In, Meta, Meta) :-
  merge_options([encoding('utf-8'),max_errors(-1)], Options1, Options2),
  load_html(In, Dom1, Options2),
  clean_dom(Dom1, Dom2).



%! html_insert_dom(+Dom:list(compound))// is det.

html_insert_dom(Dom) -->
  {with_output_to(atom(Atom), xml_write_canonical(current_output, Dom, []))},
  html(\[Atom]).




<commit_msg>ADDED: Wrapper predicate load_xml/2.<commit_after>:- module(
  xml_ext,
  [
    call_on_xml/3,      % +UriSpec, +RecordName, :Goal_1
    call_on_xml/4,      % +UriSpec, +RecordName, :Goal_1, +Options
    html_download/2,    % +UriSpec, -Dom
    html_download/3,    % +UriSpec, -Dom, +Options
    html_insert_dom//1, % +Dom
    load_xml/2          % +Source, -Dom
  ]
).

/** <module> XML extensions

@author <NAME>
@version 2016/06-2017/09
*/

:- use_module(library(atom_ext)).
:- use_module(library(c14n2)).
:- use_module(library(http/html_write)).
:- use_module(library(option)).
:- use_module(library(sgml)).
:- use_module(library(uri/uri_ext)).

:- meta_predicate
    call_on_xml(+, +, 1),
    call_on_xml(+, +, 1, +),
    call_on_xml_stream(+, 1, +, +, -).





%! call_on_xml(+UriSpec:term, +RecordName:atom, :Goal_1) is det.
%! call_on_xml(+UriSpec:term, +RecordName:atom, :Goal_1,
%!             +Options:list(compound)) is det.
%
% Call Goal_1 on an XML stream, where the argument supplied to Goal_1
% is a subtree that starts with an element called RecordName.

call_on_xml(UriSpec, RecordName, Goal_1) :-
  call_on_xml(UriSpec, RecordName, Goal_1, []).


call_on_xml(UriSpec, RecordName, Goal_1, Options) :-
  call_on_uri(UriSpec, call_on_xml_stream(RecordName, Goal_1), Options).

call_on_xml_stream(RecordName, Goal_1, In, Metadata, Metadata) :-
  b_setval(xml_stream_goal, Goal_1),
  b_setval(xml_stream_record_name, RecordName),
  setup_call_cleanup(
    new_sgml_parser(Parser, []),
    (
      set_sgml_parser(Parser, space(remove)),
      sgml_parse(Parser, [call(begin,on_begin0),source(In)])
    ),
    free_sgml_parser(Parser)
  ).

on_begin0(Elem, Attr, Parser) :-
  b_getval(xml_stream_goal, Goal_1),
  b_getval(xml_stream_record_name, Elem), !,
  sgml_parse(Parser, [document(Dom1),parse(content)]),
  xml_clean_dom(Dom1, Dom2),
  call(Goal_1, [element(Elem,Attr,Dom2)]).

xml_clean_dom([element(Elem,Attr,Dom1)|T1], [element(Elem,Attr,Dom2)|T2]) :- !,
  xml_clean_dom(Dom1, Dom2),
  xml_clean_dom(T1, T2).
xml_clean_dom([H|T1], T2) :-
  is_empty_atom(H), !,
  xml_clean_dom(T1, T2).
xml_clean_dom([H1|T1], [H2|T2]) :- !,
  atom_strip(H1, H2),
  xml_clean_dom(T1, T2).
xml_clean_dom([], []).



%! html_download(+UriSpec:term, -Dom:list(compound)) is det.
%! html_download(+UriSpec:term, -Dom:list(compound),
%!               +Options:list(compound)) is det.

html_download(UriSpec, Dom) :-
  html_download(UriSpec, Dom, []).


html_download(UriSpec, Dom, Options) :-
  call_on_uri(UriSpec, html_download_(Dom, Options), Options).

html_download_(Dom2, Options1, In, Meta, Meta) :-
  merge_options([encoding('utf-8'),max_errors(-1)], Options1, Options2),
  load_html(In, Dom1, Options2),
  clean_dom(Dom1, Dom2).



%! html_insert_dom(+Dom:list(compound))// is det.

html_insert_dom(Dom) -->
  {with_output_to(atom(Atom), xml_write_canonical(current_output, Dom, []))},
  html(\[Atom]).



%! load_xml(+Source, -Dom:list(compound)) is det.

load_xml(Source, Dom) :-
  load_xml(Source, Dom, []).





{'fraction_non_alphanumeric': 0.1717206132879046, 'fraction_numerical': 0.025212947189097105, 'mean_word_length': 4.778543307086614, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 59}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4700463', 'n_tokens_mistral': 2489, 'n_tokens_neox': 2380, 'n_words': 366}