<commit_before>// Python Tools for Visual Studio
// Copyright(c) Microsoft Corporation
// All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the License); you may not use
// this file except in compliance with the License. You may obtain a copy of the
// License at http://www.apache.org/licenses/LICENSE-2.0
//
// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
// MERCHANTABLITY OR NON-INFRINGEMENT.
//
// See the Apache Version 2.0 License for specific language governing
// permissions and limitations under the License.

using System;
using System.Collections.Generic;
using System.ComponentModel.Composition;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Web.Script.Serialization;
using Microsoft.PythonTools.Analysis;
using Microsoft.PythonTools.Interpreter;
using Microsoft.PythonTools.Parsing.Ast;
using Microsoft.PythonTools.Projects;

namespace Microsoft.PythonTools.Django.Analysis {
    [AnalysisExtensionName(Name)]
    partial class DjangoAnalyzer : IDisposable, IAnalysisExtension {
        internal const string Name = "django";
        internal readonly Dictionary<string, TagInfo> _tags = new Dictionary<string, TagInfo>();
        internal readonly Dictionary<string, TagInfo> _filters = new Dictionary<string, TagInfo>();
        internal readonly IList<DjangoUrl> _urls = new List<DjangoUrl>();
        private readonly HashSet<IPythonProjectEntry> _hookedEntries = new HashSet<IPythonProjectEntry>();
        internal readonly Dictionary<string, TemplateVariables> _templateFiles = new Dictionary<string, TemplateVariables>(StringComparer.OrdinalIgnoreCase);
        private ConditionalWeakTable<Node, ContextMarker> _contextTable = new ConditionalWeakTable<Node, ContextMarker>();
        private ConditionalWeakTable<Node, DeferredDecorator> _decoratorTable = new ConditionalWeakTable<Node, DeferredDecorator>();
        private readonly Dictionary<string, GetTemplateAnalysisValue> _templateAnalysis = new Dictionary<string, GetTemplateAnalysisValue>();
        private PythonAnalyzer _analyzer;
        internal static readonly Dictionary<string, string> _knownTags = MakeKnownTagsTable();
        internal static readonly Dictionary<string, string> _knownFilters = MakeKnownFiltersTable();
        
        public DjangoAnalyzer() {
            foreach (var tagName in _nestedEndTags) {
                _tags[tagName] = new TagInfo("", null);
            }
        }

        internal static readonly Dictionary<string, string> _nestedTags = new Dictionary<string, string>() {
            { "for", "endfor" },
            { "if", "endif" },
            { "ifequal", "endifequal" },
            { "ifnotequal", "endifnotequal" },
            { "ifchanged", "endifchanged" },
            { "autoescape", "endautoescape" },
            { "comment", "endcomment" },
            { "filter", "endfilter" },
            { "spaceless", "endspaceless" },
            { "with", "endwith" },
            { "empty", "endfor" },
            { "else", "endif" },
        };
        internal static readonly HashSet<string> _nestedEndTags = MakeNestedEndTags();
        internal static readonly HashSet<string> _nestedStartTags = MakeNestedStartTags();

        internal static class Commands {
            public const string GetTags = "getTags";
            public const string GetVariables = "getVariables";
            public const string GetFilters = "getFilters";
            public const string GetUrls = "getUrls";
            public const string GetMembers = "getMembers";

        }

        public string HandleCommand(string commandId, string body) {
            var serializer = new JavaScriptSerializer();
            Dictionary<string, HashSet<AnalysisValue>> variables;
            switch (commandId) {
                case Commands.GetTags:
                    return serializer.Serialize(_tags.Keys.ToArray());
                case Commands.GetVariables:
                    variables = GetVariablesForTemplateFile(body);
                    if (variables != null) {
                        return serializer.Serialize(variables.Keys.ToArray());
                    }

                    return "[]";
                case Commands.GetFilters:
                    Dictionary<string, string> res = new Dictionary<string, string>();
                    foreach (var filter in _filters) {
                        res[filter.Key] = filter.Value.Documentation;
                    }

                    return serializer.Serialize(res);
                case Commands.GetUrls:
                    // GroupBy + Select have the same effect as Distinct with a long EqualityComparer
                    return serializer.Serialize(_urls.GroupBy(url => url.FullName).Select(urlGroup => urlGroup.First()));
                case Commands.GetMembers:
                    string[] args = serializer.Deserialize<string[]>(body);
                    var file = args[0];
                    var varName = args[1];

                    variables = GetVariablesForTemplateFile(file);
                    HashSet<AnalysisValue> values;
                    IProjectEntry projEntry;
                    if (_analyzer.TryGetProjectEntryByPath(file, out projEntry)) {
                        var context = projEntry.AnalysisContext;

                        if (variables != null && variables.TryGetValue(varName, out values)) {
                            var newTags = new Dictionary<string, PythonMemberType>();
                            foreach (var member in values.SelectMany(item => item.GetAllMembers(context))) {
                                string name = member.Key;
                                PythonMemberType type, newType = GetMemberType(member.Value);

                                if (!newTags.TryGetValue(name, out type)) {
                                    newTags[name] = newType;
                                } else if (type != newType && type != PythonMemberType.Unknown && newType != PythonMemberType.Unknown) {
                                    newTags[name] = PythonMemberType.Multiple;
                                }
                            }

                            var dict = newTags.ToDictionary(x => x.Key, x => x.Value.ToString().ToLowerInvariant());
                            return serializer.Serialize(dict);
                        }
                    }
                    return "{}";
                default:
                    return String.Empty;
            }
        }

        private static PythonMemberType GetMemberType(IAnalysisSet values) {
            PythonMemberType newType = PythonMemberType.Unknown;
            foreach (var value in values) {
                if (value.MemberType == newType) {
                    continue;
                } else if (newType == PythonMemberType.Unknown) {
                    newType = value.MemberType;
                } else {
                    newType = PythonMemberType.Multiple;
                    break;
                }
            }
            return newType;
        }

        public void Register(PythonAnalyzer analyzer) {
            if (analyzer == null) {
                throw new ArgumentNullException("analyzer");
            }

            _tags.Clear();
            _filters.Clear();
            _urls.Clear();
            foreach (var entry in _hookedEntries) {
                entry.OnNewParseTree -= OnNewParseTree;
            }
            _hookedEntries.Clear();
            _templateAnalysis.Clear();
            _templateFiles.Clear();
            _contextTable = new ConditionalWeakTable<Node, ContextMarker>();
            _decoratorTable = new ConditionalWeakTable<Node, DeferredDecorator>();

            foreach (var keyValue in _knownTags) {
                _tags[keyValue.Key] = new TagInfo(keyValue.Value, null);
            }
            foreach (var keyValue in _knownFilters) {
                _filters[keyValue.Key] = new TagInfo(keyValue.Value, null);
            }

            HookAnalysis(analyzer);
            _analyzer = analyzer;
        }

        private void OnNewParseTree(object sender, EventArgs e) {
            var entry = sender as IPythonProjectEntry;
            if (entry != null && _hookedEntries.Remove(entry)) {
                var removeTags = _tags.Where(kv => kv.Value.Entry == entry).Select(kv => kv.Key).ToList();
                var removeFilters = _filters.Where(kv => kv.Value.Entry == entry).Select(kv => kv.Key).ToList();
                foreach (var key in removeTags) {
                    _tags.Remove(key);
                }
                foreach (var key in removeFilters) {
                    _filters.Remove(key);
                }
            }
        }

        private void HookAnalysis(PythonAnalyzer analyzer) {
            analyzer.SpecializeFunction("django.template.loader", "render_to_string", RenderToStringProcessor, true);
            analyzer.SpecializeFunction("django.shortcuts", "render_to_response", RenderToStringProcessor, true);
            analyzer.SpecializeFunction("django.shortcuts", "render", RenderProcessor, true);
            analyzer.SpecializeFunction("django.contrib.gis.shortcuts", "render_to_kml", RenderToStringProcessor, true);
            analyzer.SpecializeFunction("django.contrib.gis.shortcuts", "render_to_kmz", RenderToStringProcessor, true);
            analyzer.SpecializeFunction("django.contrib.gis.shortcuts", "render_to_text", RenderToStringProcessor, true);

            analyzer.SpecializeFunction("django.template.Library", "filter", FilterProcessor, true);
            analyzer.SpecializeFunction("django.template.Library", "filter_function", FilterProcessor, true);

            analyzer.SpecializeFunction("django.template.Library", "tag", TagProcessor, true);
            analyzer.SpecializeFunction("django.template.Library", "tag_function", TagProcessor, true);
            analyzer.SpecializeFunction("django.template.Library", "assignment_tag", TagProcessor, true);
            analyzer.SpecializeFunction("django.template.Library", "simple_tag", TagProcessor, true);

            // Django >= 1.9
            analyzer.SpecializeFunction("django.template.library", "import_library", "django.template.library.Library", true);

            // Django < 1.9
            analyzer.SpecializeFunction("django.template.base", "import_library", "django.template.base.Library", true);

            analyzer.SpecializeFunction("django.template.base.Parser", "parse", ParseProcessor, true);

            analyzer.SpecializeFunction("django.template.loader", "get_template", GetTemplateProcessor, true);
            analyzer.SpecializeFunction("django.template.context", "Context", ContextClassProcessor, true);
            analyzer.SpecializeFunction("django.template", "RequestContext", RequestContextClassProcessor, true);
            analyzer.SpecializeFunction("django.template.context", "RequestContext", RequestContextClassProcessor, true);
            analyzer.SpecializeFunction("django.template.base.Template", "render", TemplateRenderProcessor, true);

            // View specializers
            analyzer.SpecializeFunction("django.views.generic.detail.DetailView", "as_view", DetailViewProcessor, true);
            analyzer.SpecializeFunction("django.views.generic.DetailView", "as_view", DetailViewProcessor, true);
            analyzer.SpecializeFunction("django.views.generic.list.ListView", "as_view", ListViewProcessor, true);
            analyzer.SpecializeFunction("django.views.generic.ListView", "as_view", ListViewProcessor, true);

            // Urls specializers
            analyzer.SpecializeFunction("django.conf.urls", "url", UrlProcessor, true);<commit_msg>Update Django analyzer for Django 2.* path and url functions.<commit_after>// Python Tools for Visual Studio
// Copyright(c) Microsoft Corporation
// All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the License); you may not use
// this file except in compliance with the License. You may obtain a copy of the
// License at http://www.apache.org/licenses/LICENSE-2.0
//
// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
// MERCHANTABLITY OR NON-INFRINGEMENT.
//
// See the Apache Version 2.0 License for specific language governing
// permissions and limitations under the License.

using System;
using System.Collections.Generic;
using System.ComponentModel.Composition;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Web.Script.Serialization;
using Microsoft.PythonTools.Analysis;
using Microsoft.PythonTools.Interpreter;
using Microsoft.PythonTools.Parsing.Ast;
using Microsoft.PythonTools.Projects;

namespace Microsoft.PythonTools.Django.Analysis {
    [AnalysisExtensionName(Name)]
    partial class DjangoAnalyzer : IDisposable, IAnalysisExtension {
        internal const string Name = "django";
        internal readonly Dictionary<string, TagInfo> _tags = new Dictionary<string, TagInfo>();
        internal readonly Dictionary<string, TagInfo> _filters = new Dictionary<string, TagInfo>();
        internal readonly IList<DjangoUrl> _urls = new List<DjangoUrl>();
        private readonly HashSet<IPythonProjectEntry> _hookedEntries = new HashSet<IPythonProjectEntry>();
        internal readonly Dictionary<string, TemplateVariables> _templateFiles = new Dictionary<string, TemplateVariables>(StringComparer.OrdinalIgnoreCase);
        private ConditionalWeakTable<Node, ContextMarker> _contextTable = new ConditionalWeakTable<Node, ContextMarker>();
        private ConditionalWeakTable<Node, DeferredDecorator> _decoratorTable = new ConditionalWeakTable<Node, DeferredDecorator>();
        private readonly Dictionary<string, GetTemplateAnalysisValue> _templateAnalysis = new Dictionary<string, GetTemplateAnalysisValue>();
        private PythonAnalyzer _analyzer;
        internal static readonly Dictionary<string, string> _knownTags = MakeKnownTagsTable();
        internal static readonly Dictionary<string, string> _knownFilters = MakeKnownFiltersTable();
        
        public DjangoAnalyzer() {
            foreach (var tagName in _nestedEndTags) {
                _tags[tagName] = new TagInfo("", null);
            }
        }

        internal static readonly Dictionary<string, string> _nestedTags = new Dictionary<string, string>() {
            { "for", "endfor" },
            { "if", "endif" },
            { "ifequal", "endifequal" },
            { "ifnotequal", "endifnotequal" },
            { "ifchanged", "endifchanged" },
            { "autoescape", "endautoescape" },
            { "comment", "endcomment" },
            { "filter", "endfilter" },
            { "spaceless", "endspaceless" },
            { "with", "endwith" },
            { "empty", "endfor" },
            { "else", "endif" },
        };
        internal static readonly HashSet<string> _nestedEndTags = MakeNestedEndTags();
        internal static readonly HashSet<string> _nestedStartTags = MakeNestedStartTags();

        internal static class Commands {
            public const string GetTags = "getTags";
            public const string GetVariables = "getVariables";
            public const string GetFilters = "getFilters";
            public const string GetUrls = "getUrls";
            public const string GetMembers = "getMembers";

        }

        public string HandleCommand(string commandId, string body) {
            var serializer = new JavaScriptSerializer();
            Dictionary<string, HashSet<AnalysisValue>> variables;
            switch (commandId) {
                case Commands.GetTags:
                    return serializer.Serialize(_tags.Keys.ToArray());
                case Commands.GetVariables:
                    variables = GetVariablesForTemplateFile(body);
                    if (variables != null) {
                        return serializer.Serialize(variables.Keys.ToArray());
                    }

                    return "[]";
                case Commands.GetFilters:
                    Dictionary<string, string> res = new Dictionary<string, string>();
                    foreach (var filter in _filters) {
                        res[filter.Key] = filter.Value.Documentation;
                    }

                    return serializer.Serialize(res);
                case Commands.GetUrls:
                    // GroupBy + Select have the same effect as Distinct with a long EqualityComparer
                    return serializer.Serialize(_urls.GroupBy(url => url.FullName).Select(urlGroup => urlGroup.First()));
                case Commands.GetMembers:
                    string[] args = serializer.Deserialize<string[]>(body);
                    var file = args[0];
                    var varName = args[1];

                    variables = GetVariablesForTemplateFile(file);
                    HashSet<AnalysisValue> values;
                    IProjectEntry projEntry;
                    if (_analyzer.TryGetProjectEntryByPath(file, out projEntry)) {
                        var context = projEntry.AnalysisContext;

                        if (variables != null && variables.TryGetValue(varName, out values)) {
                            var newTags = new Dictionary<string, PythonMemberType>();
                            foreach (var member in values.SelectMany(item => item.GetAllMembers(context))) {
                                string name = member.Key;
                                PythonMemberType type, newType = GetMemberType(member.Value);

                                if (!newTags.TryGetValue(name, out type)) {
                                    newTags[name] = newType;
                                } else if (type != newType && type != PythonMemberType.Unknown && newType != PythonMemberType.Unknown) {
                                    newTags[name] = PythonMemberType.Multiple;
                                }
                            }

                            var dict = newTags.ToDictionary(x => x.Key, x => x.Value.ToString().ToLowerInvariant());
                            return serializer.Serialize(dict);
                        }
                    }
                    return "{}";
                default:
                    return String.Empty;
            }
        }

        private static PythonMemberType GetMemberType(IAnalysisSet values) {
            PythonMemberType newType = PythonMemberType.Unknown;
            foreach (var value in values) {
                if (value.MemberType == newType) {
                    continue;
                } else if (newType == PythonMemberType.Unknown) {
                    newType = value.MemberType;
                } else {
                    newType = PythonMemberType.Multiple;
                    break;
                }
            }
            return newType;
        }

        public void Register(PythonAnalyzer analyzer) {
            if (analyzer == null) {
                throw new ArgumentNullException("analyzer");
            }

            _tags.Clear();
            _filters.Clear();
            _urls.Clear();
            foreach (var entry in _hookedEntries) {
                entry.OnNewParseTree -= OnNewParseTree;
            }
            _hookedEntries.Clear();
            _templateAnalysis.Clear();
            _templateFiles.Clear();
            _contextTable = new ConditionalWeakTable<Node, ContextMarker>();
            _decoratorTable = new ConditionalWeakTable<Node, DeferredDecorator>();

            foreach (var keyValue in _knownTags) {
                _tags[keyValue.Key] = new TagInfo(keyValue.Value, null);
            }
            foreach (var keyValue in _knownFilters) {
                _filters[keyValue.Key] = new TagInfo(keyValue.Value, null);
            }

            HookAnalysis(analyzer);
            _analyzer = analyzer;
        }

        private void OnNewParseTree(object sender, EventArgs e) {
            var entry = sender as IPythonProjectEntry;
            if (entry != null && _hookedEntries.Remove(entry)) {
                var removeTags = _tags.Where(kv => kv.Value.Entry == entry).Select(kv => kv.Key).ToList();
                var removeFilters = _filters.Where(kv => kv.Value.Entry == entry).Select(kv => kv.Key).ToList();
                foreach (var key in removeTags) {
                    _tags.Remove(key);
                }
                foreach (var key in removeFilters) {
                    _filters.Remove(key);
                }
            }
        }

        private void HookAnalysis(PythonAnalyzer analyzer) {
            analyzer.SpecializeFunction("django.template.loader", "render_to_string", RenderToStringProcessor, true);
            analyzer.SpecializeFunction("django.shortcuts", "render_to_response", RenderToStringProcessor, true);
            analyzer.SpecializeFunction("django.shortcuts", "render", RenderProcessor, true);
            analyzer.SpecializeFunction("django.contrib.gis.shortcuts", "render_to_kml", RenderToStringProcessor, true);
            analyzer.SpecializeFunction("django.contrib.gis.shortcuts", "render_to_kmz", RenderToStringProcessor, true);
            analyzer.SpecializeFunction("django.contrib.gis.shortcuts", "render_to_text", RenderToStringProcessor, true);

            analyzer.SpecializeFunction("django.template.Library", "filter", FilterProcessor, true);
            analyzer.SpecializeFunction("django.template.Library", "filter_function", FilterProcessor, true);

            analyzer.SpecializeFunction("django.template.Library", "tag", TagProcessor, true);
            analyzer.SpecializeFunction("django.template.Library", "tag_function", TagProcessor, true);
            analyzer.SpecializeFunction("django.template.Library", "assignment_tag", TagProcessor, true);
            analyzer.SpecializeFunction("django.template.Library", "simple_tag", TagProcessor, true);

            // Django >= 1.9
            analyzer.SpecializeFunction("django.template.library", "import_library", "django.template.library.Library", true);

            // Django < 1.9
            analyzer.SpecializeFunction("django.template.base", "import_library", "django.template.base.Library", true);

            analyzer.SpecializeFunction("django.template.base.Parser", "parse", ParseProcessor, true);

            analyzer.SpecializeFunction("django.template.loader", "get_template", GetTemplateProcessor, true);
            analyzer.SpecializeFunction("django.template.context", "Context", ContextClassProcessor, true);
            analyzer.SpecializeFunction("django.template", "RequestContext", RequestContextClassProcessor, true);
            analyzer.SpecializeFunction("django.template.context", "RequestContext", RequestContextClassProcessor, true);
            analyzer.SpecializeFunction("django.template.base.Template", "render", TemplateRenderProcessor, true);

            // View specializers
            analyzer.SpecializeFunction("django.views.generic.detail.DetailView", "as_view", DetailViewProcessor, true);
            analyzer.SpecializeFunction("django.views.generic.DetailView", "as_view", DetailViewProcessor, true);
            analyzer.SpecializeFunction("django.views.generic.list.ListView", "as_view", ListViewProcessor, true);
            analyzer.SpecializeFunction("django.views.generic.ListView", "as_view", ListViewProcessor, true);

            // Urls specializers
            analyzer.SpecializeFunction("django.conf.urls", "url", UrlProcessor, true);
            analyzer.SpecializeFunction("django.urls", "url", UrlProcessor, true);
            analyzer.SpecializeFunction("django.urls", "path", UrlProcessor, true);
{'fraction_non_alphanumeric': 0.09823093176451092, 'fraction_numerical': 0.0010357542362348262, 'mean_word_length': 2.2796195652173914, 'pattern_counts': {'":': 0, '<': 67, '<?xml version=': 0, '>': 85, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5621328', 'n_tokens_mistral': 5881, 'n_tokens_neox': 5694, 'n_words': 1352}