<commit_before>﻿using Antlr4.Runtime;
using Antlr4.Runtime.Tree;
using Antlr4Intellisense.Syntax;
using Antlr4Intellisense.Syntax.InternalSyntax;
using MetaDslx.CodeAnalysis;
using MetaDslx.CodeAnalysis.Syntax;
using MetaDslx.CodeAnalysis.Syntax.InternalSyntax;
using MetaDslx.Languages.Antlr4Roslyn.Syntax.InternalSyntax;
using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.Text;
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Text;
using System.Threading;

namespace Antlr4Intellisense
{
    public class Program
    {
        public static void Main(string[] args)
        {
            //*/        0123456789012
            var text = "var a = 1 + 2\r\nvar b = a+1\r\n";
            var change = ImmutableArray<TextChangeRange>.Empty;

            var lexer = SandyLanguage.Instance.InternalSyntaxFactory.CreateLexer(SourceText.From(text), null, null);
            /*LexerMode mode = null;
            InternalSyntaxToken token;
            do
            {
                token = lexer.Lex(ref mode);
                Console.WriteLine(token.KindText + ": " + token.Text);
            } while (token.Kind != SandySyntaxKind.Eof);
            Console.WriteLine("Lookahead: {0}..{1}", lexer.MinLookahead, lexer.MaxLookahead);*/

            var parser = SandyLanguage.Instance.InternalSyntaxFactory.CreateParser(SourceText.From(text), null, null, null);
            var tree = parser.Parse();<commit_msg>Revert "Merge branch 'master' of github.com:balazssimon/meta-cs"<commit_after>﻿using Antlr4.Runtime;
using Antlr4.Runtime.Tree;
using Antlr4Intellisense.Syntax;
using Antlr4Intellisense.Syntax.InternalSyntax;
using MetaDslx.CodeAnalysis;
using MetaDslx.CodeAnalysis.Syntax;
using MetaDslx.CodeAnalysis.Syntax.InternalSyntax;
using MetaDslx.Languages.Antlr4Roslyn.Syntax.InternalSyntax;
using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.Text;
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Text;
using System.Threading;

namespace Antlr4Intellisense
{
    /*public class IncrementalSandyParser : IncrementalAntlr4Parser
    {
        public IncrementalSandyParser(Language language, SourceText text, LanguageParseOptions options, ImmutableArray<TextChangeRange> changes, SyntaxNode oldTree, CancellationToken cancellationToken = default) 
            : base(language, text, options, oldTree, changes, cancellationToken)
        {
        }

        protected override ParserRuleContext Antlr4ParseMainRule()
        {
            var parser = (SandyParser)this.Antlr4Parser;
            var sandyFile = parser.sandyFile();
            return sandyFile;
        }

        public IParseTree ParseTree()
        {
            return Antlr4ParseMainRule();
        }

        protected override GreenNode GetOrCreateGreenNode(ParserRuleContext context)
        {
            return null;
        }

        private class A
        {
            private void Foo(IncrementalSandyParser p)
            {
                p.cancellationToken.ThrowIfCancellationRequested();
            }
        }
    }*/

    public class Program
    {
        public static void Main(string[] args)
        {
            //*/        0123456789012
            var text = "var a = 1 + 2\r\nvar b = a+1\r\n";
            var change = ImmutableArray<TextChangeRange>.Empty;

            var lexer = SandyLanguage.Instance.InternalSyntaxFactory.CreateLexer(SourceText.From(text), null, null);
            /*LexerMode mode = null;
            InternalSyntaxToken token;
            do
            {
                token = lexer.Lex(ref mode);
                Console.WriteLine(token.KindText + ": " + token.Text);
            } while (token.Kind != SandySyntaxKind.Eof);
            Console.WriteLine("Lookahead: {0}..{1}", lexer.MinLookahead, lexer.MaxLookahead);*/

            var parser = SandyLanguage.Instance.InternalSyntaxFactory.CreateParser(SourceText.From(text), null, null, null);
            var tree = parser.Parse();
{'fraction_non_alphanumeric': 0.0932349323493235, 'fraction_numerical': 0.012792127921279213, 'mean_word_length': 2.70985401459854, 'pattern_counts': {'":': 2, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6806645', 'n_tokens_mistral': 1178, 'n_tokens_neox': 1121, 'n_words': 232}