<commit_before>using System;
using System.Collections.Generic;
using System.Linq;
using DiGraph;
using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.CSharp;
using Spg.ExampleRefactoring.AST;
using Spg.ExampleRefactoring.Comparator;
using Spg.ExampleRefactoring.Digraph;
using Spg.ExampleRefactoring.Expression;
using Spg.ExampleRefactoring.Partition;
using Spg.ExampleRefactoring.Position;
using Spg.ExampleRefactoring.Setting;
using Spg.ExampleRefactoring.Tok;
using Spg.LocationRefactor.Predicate;
using Spg.LocationRefactoring.Tok;
using System.IO;

namespace Spg.ExampleRefactoring.Synthesis
{
    /// <summary>
    /// Class to generate programs
    /// </summary>
    public class ASTProgram
    {
        /// <summary>
        /// Setting for AST synthesis computation
        /// </summary>
        /// <returns>Setting</returns>
        public SynthesizerSetting Setting { get; set; }

        /// <summary>
        /// Previous computed token sequences
        /// </summary>
        private readonly Dictionary<ListNode, List<TokenSeq>> _computed = new Dictionary<ListNode, List<TokenSeq>>();


        /// <summary>
        /// Dynamic tokens and occurrences
        /// </summary>
        /// <returns></returns>
        public Dictionary<DymToken, List<DymToken>> Dict { get; set; }

        /// <summary>
        /// Constructor
        /// </summary>
        public ASTProgram()
        {
            Dict = new Dictionary<DymToken, List<DymToken>>();
            this.Setting = new SynthesizerSetting { Deviation = 2, ConsiderConstrStr = true };
        }

        /// <summary>
        /// Constructor
        /// </summary>
        /// <param name="setting">Configuration setting</param>
        /// <param name="examples">Example list</param>
        /// <exception cref="Exception">Argument exception</exception>
        public ASTProgram(SynthesizerSetting setting, List<Tuple<ListNode, ListNode>> examples)
        {
            if (examples == null || examples.Count == 0) { throw new ArgumentException("Examples cannot be null or empty"); }

            this.Setting = setting;
            Dict = new Dictionary<DymToken, List<DymToken>>();

            if (setting.DynamicTokens)
            {
                CreateDymTokens(examples, setting._getFullyQualifiedName);
            }
        }

        ///// <summary>
        ///// Generate synthesized programs
        ///// </summary>
        ///// <param name="examples">Example set</param>
        ///// <returns>Synthesized program list</returns>
        //public List<SynthesizedProgram> GenerateStringProgram(List<Tuple<ListNode, ListNode>> examples)
        //{
        //    if (examples == null || examples.Count == 0) { throw new ArgumentException("Examples cannot be null or empty"); }

        //    List<SynthesizedProgram> validated = new List<SynthesizedProgram>();
        //    if (OutputIsEmpty(examples))
        //    {
        //        return GetEmptyProgram();
        //    }

        //    List<Dag> dags = Dags(examples);

        //    IntersectManager intManager = new IntersectManager();

        //    Dag T = intManager.Intersect(dags);

        //    //remove
        //    PartitionManager pManager = new PartitionManager();
        //    pManager.GeneratePartition(dags);
        //    //remove

        //    if (T == null)
        //    {
        //        dags = Dags(examples, false);
        //        T = intManager.Intersect(dags);

        //        //remove
        //        pManager = new PartitionManager();
        //        pManager.GeneratePartition(dags);
        //        //remove
        //    }

        //    if (T == null)
        //    {
        //        Setting.Deviation = 1;
        //        Console.WriteLine("Cannot generate programs for the defaut deviation. Setting deviation for 1.");
        //        dags = Dags(examples);
        //        T = intManager.Intersect(dags);

        //        //remove
        //        pManager = new PartitionManager();
        //        pManager.GeneratePartition(dags);
        //        //remove
        //    }

        //    ExpressionManager expmanager = new ExpressionManager();
        //    expmanager.FilterExpressions(T, examples);

        //    Clear(T);

        //    BreadthFirstDirectedPaths bfs = new BreadthFirstDirectedPaths(T.dag, T.Init.Id);
        //    double dist = bfs.DistTo(T.End.Id);
        //    Console.WriteLine(dist);

        //    List<Vertex> solutions = new List<Vertex>();

        //    foreach (string s in bfs.PathTo(T.End.Id))
        //    {
        //        solutions.Add(T.Vertexes[s]);
        //    }

        //    SynthesisManager manager = new SynthesisManager(Setting);
        //    SynthesizedProgram valid = manager.FilterASTPrograms(T.Mapping, solutions, examples);
        //    Console.WriteLine(valid);

        //    validated.Add(valid);
        //    return validated;
        //}

        /// <summary>
        /// Generate synthesized programs
        /// </summary>
        /// <param name="examples">Example set</param>
        /// <param name="boundary">Determine if to generate the program the tool needs to use boundaries.</param>
        /// <returns>Synthesized program list</returns>
        public List<SynthesizedProgram> GenerateStringProgram(List<Tuple<ListNode, ListNode>> examples, bool boundary = true)
        {
            if (examples == null || examples.Count == 0) { throw new ArgumentException("Examples cannot be null or empty"); }

            List<SynthesizedProgram> validated = new List<SynthesizedProgram>();
            if (OutputIsEmpty(examples))
            {
                return GetEmptyProgram();
            }

            Dictionary<Dag, List<Tuple<ListNode, ListNode>>> dags = Dags(examples, boundary);
            PartitionManager pManager = new PartitionManager();
            Dictionary<Dag, List<Tuple<ListNode, ListNode>>> Ts = pManager.GeneratePartition(dags);


            if (examples.Count()  > 1  && Ts.Count == examples.Count && boundary)
            {
                //Setting.Deviation = 1;
                return GenerateStringProgram(examples, false);
            }<commit_msg>uncommend line that search for all patterns that converts an input to an output.<commit_after>using System;
using System.Collections.Generic;
using System.Linq;
using DiGraph;
using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.CSharp;
using Spg.ExampleRefactoring.AST;
using Spg.ExampleRefactoring.Comparator;
using Spg.ExampleRefactoring.Digraph;
using Spg.ExampleRefactoring.Expression;
using Spg.ExampleRefactoring.Partition;
using Spg.ExampleRefactoring.Position;
using Spg.ExampleRefactoring.Setting;
using Spg.ExampleRefactoring.Tok;
using Spg.LocationRefactor.Predicate;
using Spg.LocationRefactoring.Tok;
using System.IO;

namespace Spg.ExampleRefactoring.Synthesis
{
    /// <summary>
    /// Class to generate programs
    /// </summary>
    public class ASTProgram
    {
        /// <summary>
        /// Setting for AST synthesis computation
        /// </summary>
        /// <returns>Setting</returns>
        public SynthesizerSetting Setting { get; set; }

        /// <summary>
        /// Previous computed token sequences
        /// </summary>
        private readonly Dictionary<ListNode, List<TokenSeq>> _computed = new Dictionary<ListNode, List<TokenSeq>>();


        /// <summary>
        /// Dynamic tokens and occurrences
        /// </summary>
        /// <returns></returns>
        public Dictionary<DymToken, List<DymToken>> Dict { get; set; }

        /// <summary>
        /// Constructor
        /// </summary>
        public ASTProgram()
        {
            Dict = new Dictionary<DymToken, List<DymToken>>();
            this.Setting = new SynthesizerSetting { Deviation = 2, ConsiderConstrStr = true };
        }

        /// <summary>
        /// Constructor
        /// </summary>
        /// <param name="setting">Configuration setting</param>
        /// <param name="examples">Example list</param>
        /// <exception cref="Exception">Argument exception</exception>
        public ASTProgram(SynthesizerSetting setting, List<Tuple<ListNode, ListNode>> examples)
        {
            if (examples == null || examples.Count == 0) { throw new ArgumentException("Examples cannot be null or empty"); }

            this.Setting = setting;
            Dict = new Dictionary<DymToken, List<DymToken>>();

            if (setting.DynamicTokens)
            {
                CreateDymTokens(examples, setting._getFullyQualifiedName);
            }
        }

        ///// <summary>
        ///// Generate synthesized programs
        ///// </summary>
        ///// <param name="examples">Example set</param>
        ///// <returns>Synthesized program list</returns>
        //public List<SynthesizedProgram> GenerateStringProgram(List<Tuple<ListNode, ListNode>> examples)
        //{
        //    if (examples == null || examples.Count == 0) { throw new ArgumentException("Examples cannot be null or empty"); }

        //    List<SynthesizedProgram> validated = new List<SynthesizedProgram>();
        //    if (OutputIsEmpty(examples))
        //    {
        //        return GetEmptyProgram();
        //    }

        //    List<Dag> dags = Dags(examples);

        //    IntersectManager intManager = new IntersectManager();

        //    Dag T = intManager.Intersect(dags);

        //    //remove
        //    PartitionManager pManager = new PartitionManager();
        //    pManager.GeneratePartition(dags);
        //    //remove

        //    if (T == null)
        //    {
        //        dags = Dags(examples, false);
        //        T = intManager.Intersect(dags);

        //        //remove
        //        pManager = new PartitionManager();
        //        pManager.GeneratePartition(dags);
        //        //remove
        //    }

        //    if (T == null)
        //    {
        //        Setting.Deviation = 1;
        //        Console.WriteLine("Cannot generate programs for the defaut deviation. Setting deviation for 1.");
        //        dags = Dags(examples);
        //        T = intManager.Intersect(dags);

        //        //remove
        //        pManager = new PartitionManager();
        //        pManager.GeneratePartition(dags);
        //        //remove
        //    }

        //    ExpressionManager expmanager = new ExpressionManager();
        //    expmanager.FilterExpressions(T, examples);

        //    Clear(T);

        //    BreadthFirstDirectedPaths bfs = new BreadthFirstDirectedPaths(T.dag, T.Init.Id);
        //    double dist = bfs.DistTo(T.End.Id);
        //    Console.WriteLine(dist);

        //    List<Vertex> solutions = new List<Vertex>();

        //    foreach (string s in bfs.PathTo(T.End.Id))
        //    {
        //        solutions.Add(T.Vertexes[s]);
        //    }

        //    SynthesisManager manager = new SynthesisManager(Setting);
        //    SynthesizedProgram valid = manager.FilterASTPrograms(T.Mapping, solutions, examples);
        //    Console.WriteLine(valid);

        //    validated.Add(valid);
        //    return validated;
        //}

        /// <summary>
        /// Generate synthesized programs
        /// </summary>
        /// <param name="examples">Example set</param>
        /// <param name="boundary">Determine if to generate the program the tool needs to use boundaries.</param>
        /// <returns>Synthesized program list</returns>
        public List<SynthesizedProgram> GenerateStringProgram(List<Tuple<ListNode, ListNode>> examples, bool boundary = true)
        {
            if (examples == null || examples.Count == 0) { throw new ArgumentException("Examples cannot be null or empty"); }

            List<SynthesizedProgram> validated = new List<SynthesizedProgram>();
            if (OutputIsEmpty(examples))
            {
                return GetEmptyProgram();
            }

            Dictionary<Dag, List<Tuple<ListNode, ListNode>>> dags = Dags(examples, boundary);
            PartitionManager pManager = new PartitionManager();
            Dictionary<Dag, List<Tuple<ListNode, ListNode>>> Ts = pManager.GeneratePartition(dags);


            if (examples.Count() > 1 && Ts.Count == examples.Count && boundary)
            {
                //Setting.Deviation = 1;
                return GenerateStringProgram(examples, false);
            }
{'fraction_non_alphanumeric': 0.1262963260712276, 'fraction_numerical': 0.0012862770319157489, 'mean_word_length': 1.8963911525029105, 'pattern_counts': {'":': 0, '<': 137, '<?xml version=': 0, '>': 139, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1218214', 'n_tokens_mistral': 3383, 'n_tokens_neox': 3193, 'n_words': 727}