<commit_before>#region License
// Copyright 2010-2015 <NAME>
//
// This file is part of BuildAMation.
//
// BuildAMation is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// BuildAMation is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with BuildAMation.  If not, see <http://www.gnu.org/licenses/>.
#endregion // License
namespace Bam.Core
{
namespace V2
{
    using System.Linq;

    /// <summary>
    /// Extension functions
    /// </summary>
    static class Extensions
    {
        // ref: http://stackoverflow.com/questions/521687/c-sharp-foreach-with-index
        static public void Each<T>(this System.Collections.Generic.IEnumerable<T> ie, System.Action<T, int> action)
        {
            var i = 0;
            foreach (var e in ie) action(e, i++);
        }
    }

    /// <summary>
    /// Strings can contain macros and functions, which are tokenized and evaluated in this class
    /// </summary>
    public sealed class TokenizedString
    {
        public static readonly string TokenPrefix = @"$(";
        public static readonly string TokenSuffix = @")";
        private static readonly string TokenRegExPattern = @"(\$\([^)]+\))";
        private static readonly string FunctionRegExPattern = @"(\$([a-z]*)\((.+)\))";
        private static readonly string FunctionPrefix = @"$";

        private static System.Collections.Generic.List<TokenizedString> Cache = new System.Collections.Generic.List<TokenizedString>();

        private System.Collections.Generic.List<string> Tokens = null;
        private System.Collections.Generic.List<int> MacroIndices = null;
        private Module ModuleWithMacros = null;
        private string CachedJoin = null;
        private string OriginalString = null;

        static private System.Collections.Generic.IEnumerable<string> SplitToParse(string original, string regExPattern)
        {
            var matches = System.Text.RegularExpressions.Regex.Split(original, regExPattern);
            var filtered = matches.Where(item => !System.String.IsNullOrEmpty(item));
            return filtered;
        }

        private TokenizedString(string original)
        {
            this.OriginalString = original;
            var tokenized = SplitToParse(original, TokenRegExPattern);
            tokenized.Each<string>((item, index) =>
            {
                if (item.StartsWith(TokenPrefix) && item.EndsWith(TokenSuffix))
                {
                    if (null == this.MacroIndices)
                    {
                        this.MacroIndices = new System.Collections.Generic.List<int>();
                    }
                    this.MacroIndices.Add(index);
                }
            });
            this.Tokens = tokenized.ToList<string>();
        }

        private TokenizedString(string original, Module moduleWithMacros)
            : this(original)
        {
            if (null == moduleWithMacros && null != this.MacroIndices)
            {
                foreach (var tokenIndex in this.MacroIndices)
                {
                    if (!Graph.Instance.Macros.Contains(this.Tokens[tokenIndex]))
                    {
                        throw new Exception("Cannot have a tokenized string without a module");
                    }
                }
            }
            this.ModuleWithMacros = moduleWithMacros;
        }

        public static TokenizedString Create(string tokenizedString, Module macroSource)
        {
            var search = Cache.Where((ts) =>
                {
                    return ts.OriginalString == tokenizedString && ts.ModuleWithMacros == macroSource;
                });
            if (search.Count() > 0)
            {
                return search.ElementAt(0);
            }
            else
            {
                var ts = new TokenizedString(tokenizedString, macroSource);
                Cache.Add(ts);
                return ts;
            }
        }

        public string this[int index]
        {
            get
            {
                return this.Tokens[index];
            }

            set
            {
                this.Tokens[index] = value;
                if (this.MacroIndices.Contains(index))
                {
                    this.MacroIndices.Remove(index);
                }
            }
        }

        public bool IsExpanded
        {
            get
            {
                return (null == this.MacroIndices) || (0 == this.MacroIndices.Count);
            }
        }

        // safe:true = don't cache anything, and don't throw if it's not expanded
        // safe:false = cache what you can, and throw if the tokens haven't been expanded
        private string Join(bool safe)
        {
            if (!safe && !this.IsExpanded)
            {
                throw new System.Exception("TokenizedString not expanded");
            }
            if (1 == this.Tokens.Count)
            {
                return this.Tokens[0];
            }
            if (null != this.CachedJoin)
            {
                return this.CachedJoin;
            }
            var join = System.String.Join(string.Empty, this.Tokens);
            if (OSUtilities.IsWindowsHosting)
            {
                join = join.Replace('/', '\\');
            }
            else
            {
                join = join.Replace('\\', '/');
            }
            if (!safe)
            {
                this.CachedJoin = join;
            }
            return join;
        }

        public override string ToString()
        {
            return this.Join(safe: false);
        }

        public bool Empty
        {
            get
            {
                return (null == this.Tokens) || (0 == this.Tokens.Count());
            }
        }

        public static void ParseAll()
        {
            foreach (var t in Cache)
            {
                t.Parse();
            }
        }

        public void Parse()
        {
            if (this.Empty || this.IsExpanded)
            {
                return;
            }
            var orig = this.Join(safe: true);
            foreach (int index in this.MacroIndices.Reverse<int>())
            {
                var token = this.Tokens[index];
                if (Graph.Instance.Macros.Dict.ContainsKey(token))
                {
                    var value = Graph.Instance.Macros.Dict[token];
                    if (!value.IsExpanded)
                    {
                        // recursive
                        value.Parse();
                    }
                    token = value.ToString();
                }
                else if (this.ModuleWithMacros != null && this.ModuleWithMacros.Macros.Dict.ContainsKey(token))
                {
                    var value = this.ModuleWithMacros.Macros.Dict[token];
                    if (!value.IsExpanded)
                    {
                        // recursive
                        value.Parse();
                    }
                    token = value.ToString();
                }
                else if (this.ModuleWithMacros != null && null != this.ModuleWithMacros.Tool && this.ModuleWithMacros.Tool.Macros.Dict.ContainsKey(token))
                {
                    var value = this.ModuleWithMacros.Tool.Macros.Dict[token];
                    if (!value.IsExpanded)
                    {
                        // recursive
                        value.Parse();
                    }
                    token = value.ToString();
                }
                else
                {
                    // TODO: this could be due to the user not having set a property, e.g. inputpath
                    // is there a better error message that could be returned, other than this in those
                    // circumstances?
                    throw new System.Exception(System.String.Format("Unrecognized token '{0}", token));
                }
                this.Tokens[index] = token;
                this.MacroIndices.Remove(index);
            }
            if (!this.IsExpanded)
            {
                throw new System.Exception(System.String.Format("Input string could not be fully expanded"));
            }
            Core.Log.DebugMessage("Converted '{0}' to '{1}'", orig, this.ToString());
            this.EvaluateFunctions();
        }

        private void EvaluateFunctions()
        {
            var tokenized = SplitToParse(this.Join(safe: false), FunctionRegExPattern);
            var matchCount = tokenized.Count();
            if (1 == matchCount)
            {
                return;
            }
            // triplets of matches
            int matchIndex = 0;
            while (matchIndex < matchCount)
            {
                var index = matchIndex++;
                var expr = tokenized.ElementAt(index);
                if (!expr.StartsWith(FunctionPrefix))
                {
                    continue;
                }

                var functionName = tokenized.ElementAt(matchIndex++);
                var argument = tokenized.ElementAt(matchIndex++);
                var result = this.FunctionExpression(functionName, argument);
                // since there is an unsafe join above, we can do this
                this.CachedJoin = this.CachedJoin.Replace(expr, result);
            }
            Core.Log.DebugMessage("Converted further to '{0}'", this.CachedJoin);
        }

        private string FunctionExpression(string functionName, string argument)
        {
            switch (functionName)
            {
                case "basename":
                    return System.IO.Path.GetFileNameWithoutExtension(argument);

                default:
                    throw new System.Exception("Unknown function");
            }
        }

        public bool ContainsSpace
        {
            get
            {
                if (!this.IsExpanded)
                {
                    throw new Exception("String is not yet expanded");
                }
                if (null != this.CachedJoin)
                {
                    return this.CachedJoin.Contains(' ');
                }
                else
                {
                    if (this.Tokens.Count != 1)
                    {
                        throw new Exception("Tokenized string that is expanded, but has more than one token");
                    }
                    return this.Tokens[0].Contains(' ');
                }
            }
        }
    }

    /// <summary>
    /// Container of key-values pairs representing macro replacement in strings (usually paths)
    /// </summary>
    public sealed class MacroList
    {
        public MacroList()
        {
            this.DictInternal = new System.Collections.Generic.Dictionary<string, TokenizedString>();
        }

        public void Add(string key, TokenizedString value)
        {
            if (key.StartsWith(TokenizedString.TokenPrefix) || key.EndsWith(TokenizedString.TokenSuffix))
            {
                throw new System.Exception(System.String.Format("Invalid macro key: {0}", key));
            }
            if (null == value)
            {
                throw new System.Exception("Macro value cannot be null");
            }
            var formattedKey = System.String.Format("{0}{1}{2}", TokenizedString.TokenPrefix, key, TokenizedString.TokenSuffix);
            this.DictInternal[formattedKey] = value;
        }

        private System.Collections.Generic.Dictionary<string, TokenizedString> DictInternal
        {
            get;
            set;
        }

        public System.Collections.ObjectModel.ReadOnlyDictionary<string, TokenizedString> Dict
        {
            get
            {
                return new System.Collections.ObjectModel.ReadOnlyDictionary<string, TokenizedString>(this.DictInternal);
            }
        }

        public bool Contains(string token)
        {
            return this.Dict.ContainsKey(token);
        }
    }

    /// <summary>
    /// Encapsulation of all things needed to configure a build
    /// </summary>
    sealed public class Environment
    {
        public Environment()<commit_msg>[V2] Added shortcut Macro addition when it contains no tokens<commit_after>#region License
// Copyright 2010-2015 <NAME>
//
// This file is part of BuildAMation.
//
// BuildAMation is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// BuildAMation is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with BuildAMation.  If not, see <http://www.gnu.org/licenses/>.
#endregion // License
namespace Bam.Core
{
namespace V2
{
    using System.Linq;

    /// <summary>
    /// Extension functions
    /// </summary>
    static class Extensions
    {
        // ref: http://stackoverflow.com/questions/521687/c-sharp-foreach-with-index
        static public void Each<T>(this System.Collections.Generic.IEnumerable<T> ie, System.Action<T, int> action)
        {
            var i = 0;
            foreach (var e in ie) action(e, i++);
        }
    }

    /// <summary>
    /// Strings can contain macros and functions, which are tokenized and evaluated in this class
    /// </summary>
    public sealed class TokenizedString
    {
        public static readonly string TokenPrefix = @"$(";
        public static readonly string TokenSuffix = @")";
        private static readonly string TokenRegExPattern = @"(\$\([^)]+\))";
        private static readonly string FunctionRegExPattern = @"(\$([a-z]*)\((.+)\))";
        private static readonly string FunctionPrefix = @"$";

        private static System.Collections.Generic.List<TokenizedString> Cache = new System.Collections.Generic.List<TokenizedString>();

        private System.Collections.Generic.List<string> Tokens = null;
        private System.Collections.Generic.List<int> MacroIndices = null;
        private Module ModuleWithMacros = null;
        private string CachedJoin = null;
        private string OriginalString = null;

        static private System.Collections.Generic.IEnumerable<string> SplitToParse(string original, string regExPattern)
        {
            var matches = System.Text.RegularExpressions.Regex.Split(original, regExPattern);
            var filtered = matches.Where(item => !System.String.IsNullOrEmpty(item));
            return filtered;
        }

        private TokenizedString(string original)
        {
            this.OriginalString = original;
            var tokenized = SplitToParse(original, TokenRegExPattern);
            tokenized.Each<string>((item, index) =>
            {
                if (item.StartsWith(TokenPrefix) && item.EndsWith(TokenSuffix))
                {
                    if (null == this.MacroIndices)
                    {
                        this.MacroIndices = new System.Collections.Generic.List<int>();
                    }
                    this.MacroIndices.Add(index);
                }
            });
            this.Tokens = tokenized.ToList<string>();
        }

        private TokenizedString(string original, Module moduleWithMacros)
            : this(original)
        {
            if (null == moduleWithMacros && null != this.MacroIndices)
            {
                foreach (var tokenIndex in this.MacroIndices)
                {
                    if (!Graph.Instance.Macros.Contains(this.Tokens[tokenIndex]))
                    {
                        throw new Exception("Cannot have a tokenized string without a module");
                    }
                }
            }
            this.ModuleWithMacros = moduleWithMacros;
        }

        public static TokenizedString Create(string tokenizedString, Module macroSource)
        {
            var search = Cache.Where((ts) =>
                {
                    return ts.OriginalString == tokenizedString && ts.ModuleWithMacros == macroSource;
                });
            if (search.Count() > 0)
            {
                return search.ElementAt(0);
            }
            else
            {
                var ts = new TokenizedString(tokenizedString, macroSource);
                Cache.Add(ts);
                return ts;
            }
        }

        public string this[int index]
        {
            get
            {
                return this.Tokens[index];
            }

            set
            {
                this.Tokens[index] = value;
                if (this.MacroIndices.Contains(index))
                {
                    this.MacroIndices.Remove(index);
                }
            }
        }

        public bool IsExpanded
        {
            get
            {
                return (null == this.MacroIndices) || (0 == this.MacroIndices.Count);
            }
        }

        // safe:true = don't cache anything, and don't throw if it's not expanded
        // safe:false = cache what you can, and throw if the tokens haven't been expanded
        private string Join(bool safe)
        {
            if (!safe && !this.IsExpanded)
            {
                throw new System.Exception("TokenizedString not expanded");
            }
            if (1 == this.Tokens.Count)
            {
                return this.Tokens[0];
            }
            if (null != this.CachedJoin)
            {
                return this.CachedJoin;
            }
            var join = System.String.Join(string.Empty, this.Tokens);
            if (OSUtilities.IsWindowsHosting)
            {
                join = join.Replace('/', '\\');
            }
            else
            {
                join = join.Replace('\\', '/');
            }
            if (!safe)
            {
                this.CachedJoin = join;
            }
            return join;
        }

        public override string ToString()
        {
            return this.Join(safe: false);
        }

        public bool Empty
        {
            get
            {
                return (null == this.Tokens) || (0 == this.Tokens.Count());
            }
        }

        public static void ParseAll()
        {
            foreach (var t in Cache)
            {
                t.Parse();
            }
        }

        public void Parse()
        {
            if (this.Empty || this.IsExpanded)
            {
                return;
            }
            var orig = this.Join(safe: true);
            foreach (int index in this.MacroIndices.Reverse<int>())
            {
                var token = this.Tokens[index];
                if (Graph.Instance.Macros.Dict.ContainsKey(token))
                {
                    var value = Graph.Instance.Macros.Dict[token];
                    if (!value.IsExpanded)
                    {
                        // recursive
                        value.Parse();
                    }
                    token = value.ToString();
                }
                else if (this.ModuleWithMacros != null && this.ModuleWithMacros.Macros.Dict.ContainsKey(token))
                {
                    var value = this.ModuleWithMacros.Macros.Dict[token];
                    if (!value.IsExpanded)
                    {
                        // recursive
                        value.Parse();
                    }
                    token = value.ToString();
                }
                else if (this.ModuleWithMacros != null && null != this.ModuleWithMacros.Tool && this.ModuleWithMacros.Tool.Macros.Dict.ContainsKey(token))
                {
                    var value = this.ModuleWithMacros.Tool.Macros.Dict[token];
                    if (!value.IsExpanded)
                    {
                        // recursive
                        value.Parse();
                    }
                    token = value.ToString();
                }
                else
                {
                    // TODO: this could be due to the user not having set a property, e.g. inputpath
                    // is there a better error message that could be returned, other than this in those
                    // circumstances?
                    throw new System.Exception(System.String.Format("Unrecognized token '{0}", token));
                }
                this.Tokens[index] = token;
                this.MacroIndices.Remove(index);
            }
            if (!this.IsExpanded)
            {
                throw new System.Exception(System.String.Format("Input string could not be fully expanded"));
            }
            Core.Log.DebugMessage("Converted '{0}' to '{1}'", orig, this.ToString());
            this.EvaluateFunctions();
        }

        private void EvaluateFunctions()
        {
            var tokenized = SplitToParse(this.Join(safe: false), FunctionRegExPattern);
            var matchCount = tokenized.Count();
            if (1 == matchCount)
            {
                return;
            }
            // triplets of matches
            int matchIndex = 0;
            while (matchIndex < matchCount)
            {
                var index = matchIndex++;
                var expr = tokenized.ElementAt(index);
                if (!expr.StartsWith(FunctionPrefix))
                {
                    continue;
                }

                var functionName = tokenized.ElementAt(matchIndex++);
                var argument = tokenized.ElementAt(matchIndex++);
                var result = this.FunctionExpression(functionName, argument);
                // since there is an unsafe join above, we can do this
                this.CachedJoin = this.CachedJoin.Replace(expr, result);
            }
            Core.Log.DebugMessage("Converted further to '{0}'", this.CachedJoin);
        }

        private string FunctionExpression(string functionName, string argument)
        {
            switch (functionName)
            {
                case "basename":
                    return System.IO.Path.GetFileNameWithoutExtension(argument);

                default:
                    throw new System.Exception("Unknown function");
            }
        }

        public bool ContainsSpace
        {
            get
            {
                if (!this.IsExpanded)
                {
                    throw new Exception("String is not yet expanded");
                }
                if (null != this.CachedJoin)
                {
                    return this.CachedJoin.Contains(' ');
                }
                else
                {
                    if (this.Tokens.Count != 1)
                    {
                        throw new Exception("Tokenized string that is expanded, but has more than one token");
                    }
                    return this.Tokens[0].Contains(' ');
                }
            }
        }
    }

    /// <summary>
    /// Container of key-values pairs representing macro replacement in strings (usually paths)
    /// </summary>
    public sealed class MacroList
    {
        public MacroList()
        {
            this.DictInternal = new System.Collections.Generic.Dictionary<string, TokenizedString>();
        }

        public void Add(string key, TokenizedString value)
        {
            if (key.StartsWith(TokenizedString.TokenPrefix) || key.EndsWith(TokenizedString.TokenSuffix))
            {
                throw new System.Exception(System.String.Format("Invalid macro key: {0}", key));
            }
            if (null == value)
            {
                throw new System.Exception("Macro value cannot be null");
            }
            var formattedKey = System.String.Format("{0}{1}{2}", TokenizedString.TokenPrefix, key, TokenizedString.TokenSuffix);
            this.DictInternal[formattedKey] = value;
        }

        public void Add(string key, string value)
        {
            this.Add(key, TokenizedString.Create(value, null));
        }

        private System.Collections.Generic.Dictionary<string, TokenizedString> DictInternal
        {
            get;
            set;
        }

        public System.Collections.ObjectModel.ReadOnlyDictionary<string, TokenizedString> Dict
        {
            get
            {
                return new System.Collections.ObjectModel.ReadOnlyDictionary<string, TokenizedString>(this.DictInternal);
            }
        }

        public bool Contains(string token)
        {
            return this.Dict.ContainsKey(token);
        }
    }

    /// <summary>
    /// Encapsulation of all things needed to configure a build
    /// </summary>
    sealed public class Environment
    {
        public Environment()
{'fraction_non_alphanumeric': 0.09419805947283655, 'fraction_numerical': 0.002789016773382567, 'mean_word_length': 1.3620337724995362, 'pattern_counts': {'":': 2, '<': 57, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3169956', 'n_tokens_mistral': 6245, 'n_tokens_neox': 5948, 'n_words': 1535}