<commit_before>﻿// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.

using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Text.RegularExpressions;
using Microsoft.CodeAnalysis.CSharp;
using static System.String;

namespace CSharpSyntaxGenerator.Grammar
{
    internal class GrammarGenerator
    {
        private readonly ImmutableArray<TreeType> _nodes;
        private readonly Dictionary<string, List<Production>> _nameToProductions;

        public GrammarGenerator(Tree tree)
        {
            // Syntax refers to a special pseudo-element 'Modifier'.  Just synthesize that since
            // it's useful in the g4 grammar.
            var modifierKeywords = from mod in GetKinds<DeclarationModifiers>()
                                   let modKeyword = mod + "Keyword"
                                   from kind in SyntaxKinds
                                   where modKeyword == kind.ToString()
                                   select new Kind { Name = modKeyword };

            tree.Types.Add(new Node
            {
                Name = "Modifier",
                Children = { new Field { Type = SyntaxToken, Kinds = modifierKeywords.ToList() } }
            });

            _nodes = tree.Types.Where(t => t.Name != "CSharpSyntaxNode").ToImmutableArray();
            _nameToProductions = _nodes.ToDictionary(n => n.Name, _ => new List<Production>());
        }

        public string Run()
        {
            foreach (var node in _nodes)
            {
                // If this node has a base-type, then have the base-type point to this node as a
                // valid production for itself.
                if (node.Base is string nodeBase && _nameToProductions.TryGetValue(nodeBase, out var baseProductions))
                    baseProductions.Add(RuleReference(node.Name));

                if (node is Node)
                {
                    if (node.Children.Count == 0)
                        continue;

                    // Convert a rule of `a: (x | y | z)` into:
                    // a: x
                    //  | y
                    //  | z;
                    if (node.Children.Count == 1 && node.Children[0] is Field field && field.IsToken)
                    {
                        ProcessProductions(node, field.Kinds.Select(k =>
                            new List<TreeTypeChild> { new Field { Type = SyntaxToken, Kinds = { k } } }).ToArray());
                    }
                    else
                    {
                        ProcessProductions(node, node.Children);
                    }
                }
            }

            // The grammar will bottom out with certain lexical productions.  Just emit a few empty
            // productions in the grammar file indicating what's going on, and making it so that the
            // g4 file is considered legal (i.e. no rule references names of rules that don't
            // exist).

            foreach (var kind in s_lexicalTokens)
                _nameToProductions.Add(kind.ToString(), new List<Production> { new Production("/* see lexical specification */") });

            return GenerateResult();
        }

        private void ProcessProductions(TreeType node, params List<TreeTypeChild>[] productions)
            => _nameToProductions[node.Name].AddRange(productions.Select(p => ProcessChildren(p, delim: " ")));

        private string GenerateResult()
        {
            // Keep track of the rules we've emitted.  Once we've emitted a rule once, no need to do
            // it again, even if it's referenced by another rule.
            var seen = new HashSet<string>();
            var normalizedRules = new List<(string name, ImmutableArray<string> productions)>();

            // Process each major section first, followed by the entire list. That way we process
            // any rules not hit transitively from those sections.
            var orderedRules = s_majorRules.Concat(
                _nameToProductions.Keys.OrderBy(a => a, StringComparer.Ordinal));
            foreach (var rule in orderedRules)
                AddNormalizedRules(rule);

            return
@"// <auto-generated />
grammar csharp;" + Join("", normalizedRules.Select(t => Generate(t.name, t.productions)));

            void AddNormalizedRules(string name)
            {
                // Only consider the rule if it's the first time we're seeing it.
                if (seen.Add(name))
                {
                    // Order the productions alphabetically for consistency and to keep us independent
                    // from whatever ordering changes happen in Syntax.xml.
                    var sorted = _nameToProductions[name].OrderBy(v => v.Text, StringComparer.Ordinal).ToImmutableArray();
                    if (sorted.Length > 0)
                        normalizedRules.Add((Normalize(name), sorted.Select(s => s.Text).ToImmutableArray()));

                    // Now proceed in depth-first fashion through the rules the productions of this rule
                    // reference.  This helps keep related rules of these productions close by.
                    //
                    // Note: if we hit a major-section node, we don't recurse in.  This keeps us from
                    // travelling too far away, and keeps the major sections relatively cohesive.
                    var references = sorted.SelectMany(t => t.RuleReferences).Where(r => !s_majorSections.Contains(r));
                    foreach (var reference in references)
                        AddNormalizedRules(reference);
                }
            }
        }

        private static string Generate(string name, ImmutableArray<string> productions)
        {
            var sb = new StringBuilder();
            sb.AppendLine();
            sb.AppendLine();
            sb.AppendLine(name);
            sb.Append("  : ");

            if (productions.Length == 0)
            {
                throw new InvalidOperationException("Rule didn't have any productions: " + name);
            }

            sb.AppendJoin(Environment.NewLine + "  | ", productions);
            sb.AppendLine();
            sb.Append("  ;");

            return sb.ToString();
        }

        /// <summary>
        /// Returns the g4 production string for this rule based on the children it has. Also
        /// returns all the names of other rules this particular production references.
        /// </summary>
        private Production ProcessChildren(List<TreeTypeChild> children, string delim)
        {
            var result = children.Select(child => child switch
            {
                Choice c => ProcessChildren(c.Children, " | ").Parenthesize(),
                Sequence s => ProcessChildren(s.Children, " ").Parenthesize(),
                Field f => GetFieldType(f).WithSuffix(f.IsOptional ? "?" : ""),
                _ => throw new InvalidOperationException(),
            }).Where(p => p.Text.Length > 0);

            return new Production(
                Join(delim, result.Select(t => t.Text)),
                result.SelectMany(t => t.RuleReferences));
        }<commit_msg>Simplify<commit_after>﻿// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.

using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Text.RegularExpressions;
using Microsoft.CodeAnalysis.CSharp;
using static System.String;

namespace CSharpSyntaxGenerator.Grammar
{
    internal class GrammarGenerator
    {
        private readonly ImmutableArray<TreeType> _nodes;
        private readonly Dictionary<string, List<Production>> _nameToProductions;

        public GrammarGenerator(Tree tree)
        {
            // Syntax refers to a special pseudo-element 'Modifier'.  Just synthesize that since
            // it's useful in the g4 grammar.
            var modifierKeywords = from mod in GetKinds<DeclarationModifiers>()
                                   let modKeyword = mod + "Keyword"
                                   from kind in SyntaxKinds
                                   where modKeyword == kind.ToString()
                                   select new Kind { Name = modKeyword };

            tree.Types.Add(new Node
            {
                Name = "Modifier",
                Children = { new Field { Type = SyntaxToken, Kinds = modifierKeywords.ToList() } }
            });

            _nodes = tree.Types.Where(t => t.Name != "CSharpSyntaxNode").ToImmutableArray();
            _nameToProductions = _nodes.ToDictionary(n => n.Name, _ => new List<Production>());
        }

        public string Run()
        {
            foreach (var node in _nodes)
            {
                // If this node has a base-type, then have the base-type point to this node as a
                // valid production for itself.
                if (node.Base is string nodeBase && _nameToProductions.TryGetValue(nodeBase, out var baseProductions))
                    baseProductions.Add(RuleReference(node.Name));

                if (node is Node)
                {
                    if (node.Children.Count == 0)
                        continue;

                    // Convert a rule of `a: (x | y | z)` into:
                    // a: x
                    //  | y
                    //  | z;
                    if (node.Children.Count == 1 && node.Children[0] is Field field && field.IsToken)
                    {
                        ProcessProductions(node, field.Kinds.Select(k =>
                            new List<TreeTypeChild> { new Field { Type = SyntaxToken, Kinds = { k } } }).ToArray());
                    }
                    else
                    {
                        ProcessProductions(node, node.Children);
                    }
                }
            }

            // The grammar will bottom out with certain lexical productions.  Just emit a few empty
            // productions in the grammar file indicating what's going on, and making it so that the
            // g4 file is considered legal (i.e. no rule references names of rules that don't
            // exist).

            foreach (var kind in s_lexicalTokens)
                _nameToProductions.Add(kind.ToString(), new List<Production> { new Production("/* see lexical specification */") });

            return GenerateResult();
        }

        private void ProcessProductions(TreeType node, params List<TreeTypeChild>[] productions)
            => _nameToProductions[node.Name].AddRange(productions.Select(p => ProcessChildren(p, delim: " ")));

        private string GenerateResult()
        {
            // Keep track of the rules we've emitted.  Once we've emitted a rule once, no need to do
            // it again, even if it's referenced by another rule.
            var seen = new HashSet<string>();
            var normalizedRules = new List<(string name, ImmutableArray<string> productions)>();

            // Process each major section first, followed by the entire list. That way we process
            // any rules not hit transitively from those sections.
            var orderedRules = s_majorRules.Concat(
                _nameToProductions.Keys.OrderBy(a => a, StringComparer.Ordinal));
            foreach (var rule in orderedRules)
                AddNormalizedRules(rule);

            return
@"// <auto-generated />
grammar csharp;" + Join("", normalizedRules.Select(t => Generate(t.name, t.productions)));

            void AddNormalizedRules(string name)
            {
                // Only consider the rule if it's the first time we're seeing it.
                if (seen.Add(name))
                {
                    // Order the productions alphabetically for consistency and to keep us independent
                    // from whatever ordering changes happen in Syntax.xml.
                    var sorted = _nameToProductions[name].OrderBy(v => v.Text, StringComparer.Ordinal).ToImmutableArray();
                    if (sorted.Length > 0)
                        normalizedRules.Add((Normalize(name), sorted.Select(s => s.Text).ToImmutableArray()));

                    // Now proceed in depth-first fashion through the rules the productions of this rule
                    // reference.  This helps keep related rules of these productions close by.
                    //
                    // Note: if we hit a major-section node, we don't recurse in.  This keeps us from
                    // travelling too far away, and keeps the major sections relatively cohesive.
                    var references = sorted.SelectMany(t => t.RuleReferences).Where(r => !s_majorRules.Contains(r));
                    foreach (var reference in references)
                        AddNormalizedRules(reference);
                }
            }
        }

        private static string Generate(string name, ImmutableArray<string> productions)
            => Environment.NewLine + Environment.NewLine + name + Environment.NewLine + "  : " +
               Join(Environment.NewLine + "  | ", productions) + Environment.NewLine + "  ;";

        /// <summary>
        /// Returns the g4 production string for this rule based on the children it has. Also
        /// returns all the names of other rules this particular production references.
        /// </summary>
        private Production ProcessChildren(List<TreeTypeChild> children, string delim)
        {
            var result = children.Select(child => child switch
            {
                Choice c => ProcessChildren(c.Children, " | ").Parenthesize(),
                Sequence s => ProcessChildren(s.Children, " ").Parenthesize(),
                Field f => GetFieldType(f).WithSuffix(f.IsOptional ? "?" : ""),
                _ => throw new InvalidOperationException(),
            }).Where(p => p.Text.Length > 0);

            return new Production(
                Join(delim, result.Select(t => t.Text)),
                result.SelectMany(t => t.RuleReferences));
        }
{'fraction_non_alphanumeric': 0.09146001671774867, 'fraction_numerical': 0.0014628030091947619, 'mean_word_length': 1.686564371257485, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 80, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3146191', 'n_tokens_mistral': 3552, 'n_tokens_neox': 3339, 'n_words': 1088}