<commit_before>(*  Title:      Pure/Syntax/ROOT.ML
    ID:         $Id$
    Author:     <NAME> and <NAME>, TU Muenchen

This file builds the syntax module.
*)

use "pretty.ML";

use "ast.ML";
use "xgram.ML";
use "lexicon.ML";
use "extension.ML";
use "parse_tree.ML";
use "parser.ML";
use "earley0A.ML";
use "type_ext.ML";
use "sextension.ML";
use "printer.ML";
use "syntax.ML";

structure Pretty = PrettyFun();

structure Ast = AstFun(Pretty);
structure XGram = XGramFun(Ast);
structure Lexicon = LexiconFun();
structure Extension = ExtensionFun(structure XGram = XGram and Lexicon = Lexicon);
structure ParseTree = ParseTreeFun(structure Ast = Ast and Lexicon = Lexicon);
structure Parser = ParserFun(structure Symtab = Symtab and XGram = XGram
  and ParseTree = ParseTree);
structure Earley = EarleyFun(structure Symtab = Symtab and XGram = XGram
  and ParseTree = ParseTree);
structure TypeExt = TypeExtFun(structure Extension = Extension
  and Lexicon = Lexicon);
structure SExtension = SExtensionFun(structure TypeExt = TypeExt
  and Lexicon = Lexicon);
structure Printer = PrinterFun(structure Symtab = Symtab and Lexicon = Lexicon
  and TypeExt = TypeExt and SExtension = SExtension and Pretty = Pretty);
structure Syntax = SyntaxFun(structure Symtab = Symtab and TypeExt = TypeExt
  and Parser = Earley and SExtension = SExtension and Printer = Printer);


(*BasicSyntax has the most important primitives, which are made pervasive*)
signature BASIC_SYNTAX = sig include SEXTENSION0 include PRINTER0 end;
structure BasicSyntax: BASIC_SYNTAX = Syntax;
<commit_msg>added Scanner;<commit_after>(*  Title:      Pure/Syntax/ROOT.ML
    ID:         $Id$
    Author:     <NAME> and <NAME>, TU Muenchen

This file builds the syntax module.
*)

use "pretty.ML";

use "ast.ML";
use "xgram.ML";
use "lexicon.ML";
use "extension.ML";
use "parse_tree.ML";
use "parser.ML";
use "earley0A.ML";
use "type_ext.ML";
use "sextension.ML";
use "printer.ML";
use "syntax.ML";

structure Pretty = PrettyFun();

structure Ast = AstFun(Pretty);
structure XGram = XGramFun(Ast);
structure Lexicon = LexiconFun();
structure Extension = ExtensionFun(structure XGram = XGram and Lexicon = Lexicon);
structure ParseTree = ParseTreeFun(structure Ast = Ast and Lexicon = Lexicon);
structure Parser = ParserFun(structure Symtab = Symtab and XGram = XGram
  and ParseTree = ParseTree);
structure Earley = EarleyFun(structure Symtab = Symtab and XGram = XGram
  and ParseTree = ParseTree);
structure TypeExt = TypeExtFun(structure Extension = Extension
  and Lexicon = Lexicon);
structure SExtension = SExtensionFun(structure TypeExt = TypeExt
  and Lexicon = Lexicon);
structure Printer = PrinterFun(structure Symtab = Symtab and Lexicon = Lexicon
  and TypeExt = TypeExt and SExtension = SExtension and Pretty = Pretty);
structure Syntax = SyntaxFun(structure Symtab = Symtab and TypeExt = TypeExt
  and Parser = Earley and SExtension = SExtension and Printer = Printer);

structure Scanner: SCANNER = Lexicon;

(*BasicSyntax has the most important primitives, which are made pervasive*)
signature BASIC_SYNTAX = sig include SEXTENSION0 include PRINTER0 end;
structure BasicSyntax: BASIC_SYNTAX = Syntax;

{'fraction_non_alphanumeric': 0.0962566844919786, 'fraction_numerical': 0.0018873859704309531, 'mean_word_length': 4.966228893058162, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1169953', 'n_tokens_mistral': 1010, 'n_tokens_neox': 995, 'n_words': 331}