<commit_before>﻿/********************************************************************++
Copyright (c) Microsoft Corporation.  All rights reserved.
--********************************************************************/

using System;
using System.Collections;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Collections.Concurrent;
using System.Linq;
using System.Reflection;
using System.Management.Automation.Language;
using System.Management.Automation;
using System.Globalization;
using System.Text.RegularExpressions;

namespace Microsoft.Windows.Powershell.ScriptAnalyzer
{
    /// <summary>
    /// This class is used to analyze variable based on data flow
    /// </summary>
    public class VariableAnalysis : AnalysisBase
    {
        /// <summary>
        /// Constructor
        /// </summary>
        /// <param name="decorated"></param>
        public VariableAnalysis(IFlowGraph decorated) : base(decorated) { }

        private Dictionary<string, VariableAnalysisDetails> _variables;
        private readonly List<LoopGotoTargets> _loopTargets = new List<LoopGotoTargets>();
        private Dictionary<string, VariableAnalysisDetails> VariablesDictionary;
        private Dictionary<string, VariableAnalysisDetails> InternalVariablesDictionary;

        /// <summary>
        /// Return parameters of a functionmemberast or functiondefinitionast
        /// </summary>
        /// <param name="ast"></param>
        /// <param name="type"></param>
        /// <returns></returns>
        private IEnumerable<ParameterAst> FindParameters(Ast ast, Type type)
        {
            IEnumerable<Ast> parameters = ast.FindAll(item => item is ParameterAst, true);

            foreach (ParameterAst parameter in parameters)
            {
                Ast parent = parameter.Parent;
                while (parent != null)
                {
                    if (parent.GetType() == type)
                    {
                        if (parent != ast)
                        {
                            break;
                        }

                        yield return parameter;
                    }

                    parent = parent.Parent;
                }
            }
        }

        private void ProcessParameters(IEnumerable<ParameterAst> parameters)
        {
            foreach (var parameter in parameters)
            {
                var variablePath = parameter.Name.VariablePath;
                bool isSwitchOrMandatory = false;
                Type type = null;
                foreach (var paramAst in parameter.Attributes)
                {
                    if (paramAst is TypeConstraintAst)
                    {
                        if (type == null)
                        {
                            type = paramAst.TypeName.GetReflectionType();
                        }

                        if (String.Equals(paramAst.TypeName.FullName, "switch", StringComparison.OrdinalIgnoreCase))
                        {
                            isSwitchOrMandatory = true;
                        }
                    }
                    else if (paramAst is AttributeAst)
                    {
                        var args = (paramAst as AttributeAst).NamedArguments;
                        if (args != null)
                        {
                            foreach (NamedAttributeArgumentAst arg in args)
                            {
                                if (String.Equals(arg.ArgumentName, "mandatory", StringComparison.OrdinalIgnoreCase)
                                    && String.Equals(arg.Argument.Extent.Text, "$true", StringComparison.OrdinalIgnoreCase))
                                {
                                    isSwitchOrMandatory = true;
                                }
                            }
                        }
                    }
                }

                var varName = AssignmentTarget.GetUnaliasedVariableName(variablePath);
                var details = _variables[varName];
                details.Type = type ?? details.Type ?? typeof(object);

                if (parameter.DefaultValue != null)
                {
                    var assignTarget = new AssignmentTarget(varName, type);

                    if (parameter.DefaultValue is ConstantExpressionAst)
                    {
                        assignTarget.Constant = (parameter.DefaultValue as ConstantExpressionAst).Value;
                        assignTarget.Type = assignTarget.Constant == null ? typeof(object) : assignTarget.Constant.GetType();
                    }

                    Entry.AddAst(assignTarget);
                }
                else if (isSwitchOrMandatory)
                {
                    // Consider switch or mandatory parameter as already initialized
                    Entry.AddAst(new AssignmentTarget(varName, type));
                }
                else
                {
                    VariableTarget varTarget = new VariableTarget(parameter.Name);
                    varTarget.Type = details.Type;
                    Entry.AddAst(varTarget);
                }
            }
        }

        /// <summary>
        /// Used to analyze scriptbloct, functionmemberast or functiondefinitionast
        /// </summary>
        /// <param name="ast"></param>
        /// <returns></returns>
        public void AnalyzeImpl(Ast ast, VariableAnalysis outerAnalysis)
        {
            if (!(ast is ScriptBlockAst || ast is FunctionMemberAst || ast is FunctionDefinitionAst))
            {
                return;
            }

            _variables = FindAllVariablesVisitor.Visit(ast);

            Init();

            if (ast is FunctionMemberAst || ast is FunctionDefinitionAst)
            {
                IEnumerable<ParameterAst> parameters = FindParameters(ast, ast.GetType());
                if (parameters != null)
                {
                    ProcessParameters(parameters);
                }
            }
            else
            {
                ScriptBlockAst sbAst = ast as ScriptBlockAst;
                if (sbAst != null && sbAst.ParamBlock != null && sbAst.ParamBlock.Parameters != null)
                {
                    ProcessParameters(sbAst.ParamBlock.Parameters);
                }
            }

            if (ast is FunctionMemberAst)
            {
                (ast as FunctionMemberAst).Body.Visit(this.Decorator);
            }
            else if (ast is FunctionDefinitionAst)
            {
                (ast as FunctionDefinitionAst).Body.Visit(this.Decorator);
            }
            else
            {
                ast.Visit(this.Decorator);
            }

            Ast parent = ast;

            while (parent.Parent != null)
            {
                parent = parent.Parent;
            }

            List<TypeDefinitionAst> classes = parent.FindAll(item =>
                item is TypeDefinitionAst && (item as TypeDefinitionAst).IsClass, true)
                .Cast<TypeDefinitionAst>().ToList();

            if (outerAnalysis != null)
            {
                // Initialize the variables from outside
                var outerDictionary = outerAnalysis.InternalVariablesDictionary;
                foreach (var details in outerDictionary.Values)
                {
                    if (details.DefinedBlock != null)
                    {
                        var assignTarget = new AssignmentTarget(details.RealName, details.Type);
                        assignTarget.Constant = details.Constant;
                        Entry.AddFirstAst(assignTarget);
                    }
                }

                foreach (var key in _variables.Keys)
                {
                    if (outerDictionary.ContainsKey(key))
                    {
                        var outerItem = outerDictionary[key];
                        var innerItem = _variables[key];
                        innerItem.Constant = outerItem.Constant;
                        innerItem.Name = outerItem.Name;
                        innerItem.RealName = outerItem.RealName;
                        innerItem.Type = outerItem.Type;<commit_msg>Add a check to prevent crash in variable analysis<commit_after>﻿/********************************************************************++
Copyright (c) Microsoft Corporation.  All rights reserved.
--********************************************************************/

using System;
using System.Collections;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Collections.Concurrent;
using System.Linq;
using System.Reflection;
using System.Management.Automation.Language;
using System.Management.Automation;
using System.Globalization;
using System.Text.RegularExpressions;

namespace Microsoft.Windows.Powershell.ScriptAnalyzer
{
    /// <summary>
    /// This class is used to analyze variable based on data flow
    /// </summary>
    public class VariableAnalysis : AnalysisBase
    {
        /// <summary>
        /// Constructor
        /// </summary>
        /// <param name="decorated"></param>
        public VariableAnalysis(IFlowGraph decorated) : base(decorated) { }

        private Dictionary<string, VariableAnalysisDetails> _variables;
        private readonly List<LoopGotoTargets> _loopTargets = new List<LoopGotoTargets>();
        private Dictionary<string, VariableAnalysisDetails> VariablesDictionary;
        private Dictionary<string, VariableAnalysisDetails> InternalVariablesDictionary;

        /// <summary>
        /// Return parameters of a functionmemberast or functiondefinitionast
        /// </summary>
        /// <param name="ast"></param>
        /// <param name="type"></param>
        /// <returns></returns>
        private IEnumerable<ParameterAst> FindParameters(Ast ast, Type type)
        {
            IEnumerable<Ast> parameters = ast.FindAll(item => item is ParameterAst, true);

            foreach (ParameterAst parameter in parameters)
            {
                Ast parent = parameter.Parent;
                while (parent != null)
                {
                    if (parent.GetType() == type)
                    {
                        if (parent != ast)
                        {
                            break;
                        }

                        yield return parameter;
                    }

                    parent = parent.Parent;
                }
            }
        }

        private void ProcessParameters(IEnumerable<ParameterAst> parameters)
        {
            foreach (var parameter in parameters)
            {
                var variablePath = parameter.Name.VariablePath;
                bool isSwitchOrMandatory = false;
                Type type = null;
                foreach (var paramAst in parameter.Attributes)
                {
                    if (paramAst is TypeConstraintAst)
                    {
                        if (type == null)
                        {
                            type = paramAst.TypeName.GetReflectionType();
                        }

                        if (String.Equals(paramAst.TypeName.FullName, "switch", StringComparison.OrdinalIgnoreCase))
                        {
                            isSwitchOrMandatory = true;
                        }
                    }
                    else if (paramAst is AttributeAst)
                    {
                        var args = (paramAst as AttributeAst).NamedArguments;
                        if (args != null)
                        {
                            foreach (NamedAttributeArgumentAst arg in args)
                            {
                                if (String.Equals(arg.ArgumentName, "mandatory", StringComparison.OrdinalIgnoreCase)
                                    && String.Equals(arg.Argument.Extent.Text, "$true", StringComparison.OrdinalIgnoreCase))
                                {
                                    isSwitchOrMandatory = true;
                                }
                            }
                        }
                    }
                }

                var varName = AssignmentTarget.GetUnaliasedVariableName(variablePath);
                var details = _variables[varName];
                details.Type = type ?? details.Type ?? typeof(object);

                if (parameter.DefaultValue != null)
                {
                    var assignTarget = new AssignmentTarget(varName, type);

                    if (parameter.DefaultValue is ConstantExpressionAst)
                    {
                        assignTarget.Constant = (parameter.DefaultValue as ConstantExpressionAst).Value;
                        assignTarget.Type = assignTarget.Constant == null ? typeof(object) : assignTarget.Constant.GetType();
                    }

                    Entry.AddAst(assignTarget);
                }
                else if (isSwitchOrMandatory)
                {
                    // Consider switch or mandatory parameter as already initialized
                    Entry.AddAst(new AssignmentTarget(varName, type));
                }
                else
                {
                    VariableTarget varTarget = new VariableTarget(parameter.Name);
                    varTarget.Type = details.Type;
                    Entry.AddAst(varTarget);
                }
            }
        }

        /// <summary>
        /// Used to analyze scriptbloct, functionmemberast or functiondefinitionast
        /// </summary>
        /// <param name="ast"></param>
        /// <returns></returns>
        public void AnalyzeImpl(Ast ast, VariableAnalysis outerAnalysis)
        {
            if (!(ast is ScriptBlockAst || ast is FunctionMemberAst || ast is FunctionDefinitionAst))
            {
                return;
            }

            _variables = FindAllVariablesVisitor.Visit(ast);

            Init();

            if (ast is FunctionMemberAst || ast is FunctionDefinitionAst)
            {
                IEnumerable<ParameterAst> parameters = FindParameters(ast, ast.GetType());
                if (parameters != null)
                {
                    ProcessParameters(parameters);
                }
            }
            else
            {
                ScriptBlockAst sbAst = ast as ScriptBlockAst;
                if (sbAst != null && sbAst.ParamBlock != null && sbAst.ParamBlock.Parameters != null)
                {
                    ProcessParameters(sbAst.ParamBlock.Parameters);
                }
            }

            if (ast is FunctionMemberAst)
            {
                (ast as FunctionMemberAst).Body.Visit(this.Decorator);
            }
            else if (ast is FunctionDefinitionAst)
            {
                (ast as FunctionDefinitionAst).Body.Visit(this.Decorator);
            }
            else
            {
                ast.Visit(this.Decorator);
            }

            Ast parent = ast;

            while (parent.Parent != null)
            {
                parent = parent.Parent;
            }

            List<TypeDefinitionAst> classes = parent.FindAll(item =>
                item is TypeDefinitionAst && (item as TypeDefinitionAst).IsClass, true)
                .Cast<TypeDefinitionAst>().ToList();

            if (outerAnalysis != null)
            {
                // Initialize the variables from outside
                var outerDictionary = outerAnalysis.InternalVariablesDictionary;
                foreach (var details in outerDictionary.Values)
                {
                    if (details.DefinedBlock != null)
                    {
                        var assignTarget = new AssignmentTarget(details.RealName, details.Type);
                        assignTarget.Constant = details.Constant;
                        if (!_variables.ContainsKey(assignTarget.Name))
                        {
                            _variables.Add(assignTarget.Name, new VariableAnalysisDetails
                            {
                                Name = assignTarget.Name,
                                RealName = assignTarget.Name,
                                Type = assignTarget.Type
                            });
                        }
                        Entry.AddFirstAst(assignTarget);
                    }
                }

                foreach (var key in _variables.Keys)
                {
                    if (outerDictionary.ContainsKey(key))
                    {
                        var outerItem = outerDictionary[key];
                        var innerItem = _variables[key];
                        innerItem.Constant = outerItem.Constant;
                        innerItem.Name = outerItem.Name;
                        innerItem.RealName = outerItem.RealName;
                        innerItem.Type = outerItem.Type;
{'fraction_non_alphanumeric': 0.09384042992972302, 'fraction_numerical': 0.0, 'mean_word_length': 1.2803662806356046, 'pattern_counts': {'":': 0, '<': 65, '<?xml version=': 0, '>': 69, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5493541', 'n_tokens_mistral': 3830, 'n_tokens_neox': 3449, 'n_words': 798}