<commit_before>﻿// Python Tools for Visual Studio
// Copyright(c) Microsoft Corporation
// All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the License); you may not use
// this file except in compliance with the License. You may obtain a copy of the
// License at http://www.apache.org/licenses/LICENSE-2.0
//
// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
// MERCHANTABLITY OR NON-INFRINGEMENT.
//
// See the Apache Version 2.0 License for specific language governing
// permissions and limitations under the License.

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Threading;
using Microsoft.PythonTools.Analysis;
using Microsoft.PythonTools.Infrastructure;
using Microsoft.PythonTools.Parsing;
using Microsoft.PythonTools.Parsing.Ast;

namespace Microsoft.PythonTools.Interpreter.Ast {
    public sealed class AstPythonModule : IPythonModule, IProjectEntry, ILocatedMember {
        private readonly IPythonInterpreter _interpreter;
        private readonly Dictionary<object, object> _properties;
        private readonly List<string> _childModules;
        private bool _foundChildModules;
        private readonly Dictionary<string, IMember> _members;

        public static IPythonModule FromFile(
            IPythonInterpreter interpreter,
            string sourceFile,
            PythonLanguageVersion langVersion
        ) => FromFile(interpreter, sourceFile, langVersion, null);

        public static IPythonModule FromFile(
            IPythonInterpreter interpreter,
            string sourceFile,
            PythonLanguageVersion langVersion,
            string moduleFullName
        ) {
            using (var stream = new FileStream(sourceFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) {
                return FromStream(interpreter, stream, sourceFile, langVersion, moduleFullName);
            }
        }

        // Avoid hitting the filesystem, but exclude non-importable
        // paths. Ideally, we'd stop at the first path that's a known
        // search path, except we don't know search paths here.
        private static bool IsPackageCheck(string path) {
            return ModulePath.IsImportable(PathUtils.GetFileOrDirectoryName(path));
        }

        public static IPythonModule FromStream(
            IPythonInterpreter interpreter,
            Stream sourceFile,
            string fileName,
            PythonLanguageVersion langVersion
        ) => FromStream(interpreter, sourceFile, fileName, langVersion, null);

        public static IPythonModule FromStream(
            IPythonInterpreter interpreter,
            Stream sourceFile,
            string fileName,
            PythonLanguageVersion langVersion,
            string moduleFullName
        ) {
            PythonAst ast;
            using (var parser = Parser.CreateParser(sourceFile, langVersion, new ParserOptions {
                StubFile = fileName?.EndsWith(".pyi", StringComparison.OrdinalIgnoreCase) ?? false,
                Verbatim = true
            })) {
                ast = parser.ParseFile();
            }

            return new AstPythonModule(
                moduleFullName ?? ModulePath.FromFullPath(fileName, isPackage: IsPackageCheck).FullName,
                interpreter,
                ast,
                fileName
            );
        }

        internal AstPythonModule() {
            Name = string.Empty;
            Documentation = string.Empty;
            FilePath = string.Empty;
            _properties = new Dictionary<object, object>();
            _childModules = new List<string>();
            _foundChildModules = true;
            _members = new Dictionary<string, IMember>();
        }

        internal AstPythonModule(string moduleName, IPythonInterpreter interpreter, PythonAst ast, string filePath) {
            Name = moduleName;
            Documentation = ast.Documentation;
            FilePath = filePath;
            Locations = new[] { new LocationInfo(filePath, 1, 1) };
            _interpreter = interpreter;

            _properties = new Dictionary<object, object>();
            _childModules = new List<string>();
            _members = new Dictionary<string, IMember>();

            // Do not allow children of named modules
            if (!ModulePath.IsInitPyFile(FilePath)) {
                _foundChildModules = true;
            }

            var walker = new AstAnalysisWalker(interpreter, ast, this, filePath, _members, true, true);
            ast.Walk(walker);
        }

        internal void AddChildModule(string name, IPythonModule module) {
            lock (_childModules) {
                _childModules.Add(name);
            }
            lock (_members) {
                _members[name] = module;
            }
        }

        public string Name { get; }
        public string Documentation { get; }
        public string FilePath { get; }
        public PythonMemberType MemberType => PythonMemberType.Module;<commit_msg>Fixes AstAnalysis tests.<commit_after>﻿// Python Tools for Visual Studio
// Copyright(c) Microsoft Corporation
// All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the License); you may not use
// this file except in compliance with the License. You may obtain a copy of the
// License at http://www.apache.org/licenses/LICENSE-2.0
//
// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
// MERCHANTABLITY OR NON-INFRINGEMENT.
//
// See the Apache Version 2.0 License for specific language governing
// permissions and limitations under the License.

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Threading;
using Microsoft.PythonTools.Analysis;
using Microsoft.PythonTools.Infrastructure;
using Microsoft.PythonTools.Parsing;
using Microsoft.PythonTools.Parsing.Ast;

namespace Microsoft.PythonTools.Interpreter.Ast {
    public sealed class AstPythonModule : IPythonModule, IProjectEntry, ILocatedMember {
        private readonly IPythonInterpreter _interpreter;
        private readonly Dictionary<object, object> _properties;
        private readonly List<string> _childModules;
        private bool _foundChildModules;
        private readonly Dictionary<string, IMember> _members;

        public static IPythonModule FromFile(
            IPythonInterpreter interpreter,
            string sourceFile,
            PythonLanguageVersion langVersion
        ) => FromFile(interpreter, sourceFile, langVersion, null);

        public static IPythonModule FromFile(
            IPythonInterpreter interpreter,
            string sourceFile,
            PythonLanguageVersion langVersion,
            string moduleFullName
        ) {
            using (var stream = new FileStream(sourceFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) {
                return FromStream(interpreter, stream, sourceFile, langVersion, moduleFullName);
            }
        }

        // Avoid hitting the filesystem, but exclude non-importable
        // paths. Ideally, we'd stop at the first path that's a known
        // search path, except we don't know search paths here.
        private static bool IsPackageCheck(string path) {
            return ModulePath.IsImportable(PathUtils.GetFileOrDirectoryName(path));
        }

        public static IPythonModule FromStream(
            IPythonInterpreter interpreter,
            Stream sourceFile,
            string fileName,
            PythonLanguageVersion langVersion
        ) => FromStream(interpreter, sourceFile, fileName, langVersion, null);

        public static IPythonModule FromStream(
            IPythonInterpreter interpreter,
            Stream sourceFile,
            string fileName,
            PythonLanguageVersion langVersion,
            string moduleFullName
        ) {
            PythonAst ast;
            using (var parser = Parser.CreateParser(sourceFile, langVersion, new ParserOptions {
                StubFile = fileName?.EndsWith(".pyi", StringComparison.OrdinalIgnoreCase) ?? false,
                Verbatim = true
            })) {
                ast = parser.ParseFile();
            }

            return new AstPythonModule(
                moduleFullName ?? ModulePath.FromFullPath(fileName, isPackage: IsPackageCheck).FullName,
                interpreter,
                ast,
                fileName
            );
        }

        internal AstPythonModule() {
            Name = string.Empty;
            Documentation = string.Empty;
            FilePath = string.Empty;
            _properties = new Dictionary<object, object>();
            _childModules = new List<string>();
            _foundChildModules = true;
            _members = new Dictionary<string, IMember>();
        }

        internal AstPythonModule(string moduleName, IPythonInterpreter interpreter, PythonAst ast, string filePath) {
            Name = moduleName;
            Documentation = ast.Documentation;
            FilePath = filePath;
            Locations = new[] { new LocationInfo(filePath, 1, 1) };
            _interpreter = interpreter;

            _properties = new Dictionary<object, object>();
            _childModules = new List<string>();
            _members = new Dictionary<string, IMember>();

            // Do not allow children of named modules
            if (!ModulePath.IsInitPyFile(FilePath)) {
                _foundChildModules = true;
            }

            var walker = new AstAnalysisWalker(interpreter, ast, this, filePath, _members, true, true);
            ast.Walk(walker);
            walker.Complete();
        }

        internal void AddChildModule(string name, IPythonModule module) {
            lock (_childModules) {
                _childModules.Add(name);
            }
            lock (_members) {
                _members[name] = module;
            }
        }

        public string Name { get; }
        public string Documentation { get; }
        public string FilePath { get; }
        public PythonMemberType MemberType => PythonMemberType.Module;
{'fraction_non_alphanumeric': 0.07333079703252805, 'fraction_numerical': 0.0015217804831653035, 'mean_word_length': 2.292110206637445, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1215072', 'n_tokens_mistral': 2552, 'n_tokens_neox': 2489, 'n_words': 772}