<commit_before>\literate[{\tt IMPLODE-ASFIX}]
% GT -- Grammar Tools
% Copyright (C) 2000 <NAME> <<EMAIL>>
%                    <NAME> <<EMAIL>>
%                    <NAME> <<EMAIL>>
%
% This program is free software; you can redistribute it and/or modify
% it under the terms of the GNU General Public License as published by
% the Free Software Foundation; either version 2, or (at your option)
% any later version.
%
% This program is distributed in the hope that it will be useful,
% but WITHOUT ANY WARRANTY; without even the implied warranty of
% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
% GNU General Public License for more details.
%
% You should have received a copy of the GNU General Public License
% along with this program; if not, write to the Free Software
% Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
% 02111-1307, USA.

% $Id: implode-asfix.r,v 1.2 2001/06/08 09:43:07 mdejonge Exp $


	This module defines a transformation from AsFix terms produced
	by the SDF2 parser to abstract syntax terms. The
	transformation removes layout and literals, collapses lexical
	terms to strings and replaces production applications by
	applications of a derived constructor name.

	See module \verb|asfix| for a signature of AsFixTerms.

	TODO: \\
	- complete list of special constructors\\

\begin{code}
module implode-asfix
imports lib asfix options

signature
   constructors
      FlatLex      : Option
      FlatList     : Option
      RemoveLayout : Option
      FlatInj      : Option
      ReplaceAppl  : Option
      RemoveLit    : Option
      RemovePT     : Option
      RemoveSeq    : Option
      FlatAlt      : Option

      layout : String -> Layout
\end{code}

  \paragraph{Main}

	The main strategy first removes the \verb|parsetree| wrapper
	function. It then removes layout, transforms lexical trees
	into strings and context-free trees into constructor
	applications.

\begin{code}
strategies

    implode-asfix = 
      iowrap(implode, 
                Option( "--lex",    !FlatLex ) 
              + Option( "--layout", !RemoveLayout )
              + Option( "--lit",    !RemoveLit ) 
              + Option( "--alt",    !FlatAlt )
              + Option( "--appl",   !ReplaceAppl )
              + Option( "--inj",    !FlatInj )
              + Option( "--list",   !FlatList )
              + Option( "--seq",    !RemoveSeq )
              + Option( "--pt",     !RemovePT ) )

    implode =
       ?term;
       try((option-defined(FlatLex),      flat-lex));
       try((option-defined(RemoveLayout), remove-layout));
       try((option-defined(FlatList),     flat-list));
       try((option-defined(RemoveLit),    remove-lit));
       try((option-defined(FlatInj),      flat-injections));
       try((option-defined(FlatAlt),      flat-alt));
       try((option-defined(ReplaceAppl),  replace-appl));
       try((option-defined(RemoveSeq),    remove-seq));
       try((option-defined(RemovePT),     remove-pt));
       try(?term; (id, implode-asfix))

  // 1) Flatten lexical sub trees in the parse tree
  flat-lex = downup2( try(implode-lexical), try(flat-layout))
  
  // 2) Remove layout nodes from the parse tree
  remove-layout = remove-x( is-layout' )
   
  // 3) Remove literal nodes from the parse tree
  remove-lit    = 
        topdown(try(is-list;filter(not(lit(id)))))

  // 4) Flatten Alternatives: appl(alt(a,alt(b,alt(c,d))),[d]) -> alt(4,d)
  flat-alt = topdown( try(FlatAlt) )

  // 5) Replace appl nodes with their cons attribute if available.
  replace-appl = topdown(try(appl(id,id);Cns));conc-to-cons

  // 6) Flatten injections in a parse tree
  flat-injections = bottomup(try(Inj))

  // 7) Flatten list constructs 
  flat-list = topdown( try(FlatList) )

  // 8) Replace sequence by tuples
  remove-seq = topdown(try(Tuple))

  // 9) Remove parsetree term from parse tree
  remove-pt = PT
  
  // Apply all previous filters together
  implode-asfix = 
    PT
  ; rec x(try(
       implode-lexical
    <+ appl(id, filter(not(is-layout);x))<commit_msg>  * Undid accidental identification of main strategy and other strategy<commit_after>\literate[{\tt IMPLODE-ASFIX}]
% GT -- Grammar Tools
% Copyright (C) 2000 <NAME> <<EMAIL>>
%                    <NAME> <<EMAIL>>
%                    <NAME> <<EMAIL>>
%
% This program is free software; you can redistribute it and/or modify
% it under the terms of the GNU General Public License as published by
% the Free Software Foundation; either version 2, or (at your option)
% any later version.
%
% This program is distributed in the hope that it will be useful,
% but WITHOUT ANY WARRANTY; without even the implied warranty of
% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
% GNU General Public License for more details.
%
% You should have received a copy of the GNU General Public License
% along with this program; if not, write to the Free Software
% Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
% 02111-1307, USA.

% $Id: implode-asfix.r,v 1.3 2001/08/02 13:04:34 jvisser Exp $


	This module defines a transformation from AsFix terms produced
	by the SDF2 parser to abstract syntax terms. The
	transformation removes layout and literals, collapses lexical
	terms to strings and replaces production applications by
	applications of a derived constructor name.

	See module \verb|asfix| for a signature of AsFixTerms.

	TODO: \\
	- complete list of special constructors\\

\begin{code}
module implode-asfix
imports lib asfix options

signature
   constructors
      FlatLex      : Option
      FlatList     : Option
      RemoveLayout : Option
      FlatInj      : Option
      ReplaceAppl  : Option
      RemoveLit    : Option
      RemovePT     : Option
      RemoveSeq    : Option
      FlatAlt      : Option

      layout : String -> Layout
\end{code}

  \paragraph{Main}

	The main strategy first removes the \verb|parsetree| wrapper
	function. It then removes layout, transforms lexical trees
	into strings and context-free trees into constructor
	applications.

\begin{code}
strategies

    implode-asfix = 
      iowrap(implode, 
                Option( "--lex",    !FlatLex ) 
              + Option( "--layout", !RemoveLayout )
              + Option( "--lit",    !RemoveLit ) 
              + Option( "--alt",    !FlatAlt )
              + Option( "--appl",   !ReplaceAppl )
              + Option( "--inj",    !FlatInj )
              + Option( "--list",   !FlatList )
              + Option( "--seq",    !RemoveSeq )
              + Option( "--pt",     !RemovePT ) )

    implode =
       ?term;
       try((option-defined(FlatLex),      flat-lex));
       try((option-defined(RemoveLayout), remove-layout));
       try((option-defined(FlatList),     flat-list));
       try((option-defined(RemoveLit),    remove-lit));
       try((option-defined(FlatInj),      flat-injections));
       try((option-defined(FlatAlt),      flat-alt));
       try((option-defined(ReplaceAppl),  replace-appl));
       try((option-defined(RemoveSeq),    remove-seq));
       try((option-defined(RemovePT),     remove-pt));
       try(?term; (id, implodeAsfix))

  // 1) Flatten lexical sub trees in the parse tree
  flat-lex = downup2( try(implode-lexical), try(flat-layout))
  
  // 2) Remove layout nodes from the parse tree
  remove-layout = remove-x( is-layout' )
   
  // 3) Remove literal nodes from the parse tree
  remove-lit    = 
        topdown(try(is-list;filter(not(lit(id)))))

  // 4) Flatten Alternatives: appl(alt(a,alt(b,alt(c,d))),[d]) -> alt(4,d)
  flat-alt = topdown( try(FlatAlt) )

  // 5) Replace appl nodes with their cons attribute if available.
  replace-appl = topdown(try(appl(id,id);Cns));conc-to-cons

  // 6) Flatten injections in a parse tree
  flat-injections = bottomup(try(Inj))

  // 7) Flatten list constructs 
  flat-list = topdown( try(FlatList) )

  // 8) Replace sequence by tuples
  remove-seq = topdown(try(Tuple))

  // 9) Remove parsetree term from parse tree
  remove-pt = PT
  
  // Apply all previous filters together
  implodeAsfix = 
    PT
  ; rec x(try(
       implode-lexical
    <+ appl(id, filter(not(is-layout);x))
{'fraction_non_alphanumeric': 0.11157583958666503, 'fraction_numerical': 0.011563537950547422, 'mean_word_length': 2.639212175470009, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7306075', 'n_tokens_mistral': 2574, 'n_tokens_neox': 2402, 'n_words': 792}