<commit_before>﻿// ----------------------------------------------------------------------------------
//
// Copyright Microsoft Corporation
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ----------------------------------------------------------------------------------

using Microsoft.Azure.Commands.Common.Authentication.Abstractions;
using Microsoft.Azure.Commands.DataLakeStore.Properties;
using Microsoft.WindowsAzure.Commands.Utilities.Common;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Management.Automation;
using System.Net;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Azure.DataLake.Store;
using Microsoft.Azure.DataLake.Store.Acl;
using Microsoft.Azure.DataLake.Store.AclTools;
using Microsoft.Azure.DataLake.Store.FileTransfer;
using Microsoft.Azure.DataLake.Store.MockAdlsFileSystem;
using NLog;
using NLog.Config;
using NLog.Targets;
namespace Microsoft.Azure.Commands.DataLakeStore.Models
{
    public class DataLakeStoreFileSystemClient
    {

        /// <summary>
        /// The lock object
        /// </summary>
        private static readonly object ConsoleOutputLock = new object();

        private readonly IAzureContext _context;
        private readonly Random _uniqueActivityIdGenerator;
        private const int MaxConnectionLimit = 1000;
        private const long NeverExpireValue = 253402300800000;
        internal const int ImportExportMaxThreads = 256;
        private readonly LoggingConfiguration _adlsLoggerConfig;
        private readonly bool _isDebugEnabled;
        private const int DebugMessageFlushThreshold = 500;
        #region Constructors

        static DataLakeStoreFileSystemClient()
        {
            // Registering the custom target class
            Target.Register<AdlsLoggerTarget>("AdlsLogger"); //generic
            LogManager.ReconfigExistingLoggers();
        }

        public DataLakeStoreFileSystemClient(IAzureContext context)
        {
            if (context != null)
            {
                _context = context;
            }
            else
            {
                throw new ApplicationException(Resources.InvalidDefaultSubscription);
            }
            _uniqueActivityIdGenerator = new Random();

            // We need to override the default .NET value for max connections to a host to our number of threads, if necessary.
            // Otherwise we won't achieve the parallelism we want.
            // This is also required before the first call on the data lake store client.
            ServicePointManager.DefaultConnectionLimit = Math.Max(MaxConnectionLimit,
                ServicePointManager.DefaultConnectionLimit);
            ServicePointManager.Expect100Continue = false;
            ServicePointManager.UseNagleAlgorithm = false;
        }

        public DataLakeStoreFileSystemClient(IAzureContext context, DataLakeStoreFileSystemCmdletBase cmdlet) : this(context)
        {
            bool containsDebug = cmdlet.MyInvocation.BoundParameters.ContainsKey("Debug");
            if (containsDebug)
            {
                _isDebugEnabled = ((SwitchParameter) cmdlet.MyInvocation.BoundParameters["Debug"]).ToBool();
            }
            else
            {
                var debugPreference = cmdlet.GetVariableValue("DebugPreference") as string;
                if (debugPreference != null)
                {
                    _isDebugEnabled = !debugPreference.Equals("SilentlyContinue");
                }
            }
            
            if (_isDebugEnabled)
            {
                _adlsLoggerConfig = new LoggingConfiguration();
                // Custom target that logs the debug messages from the SDK to the powershell framework's debug message queue
                var adlsTarget = new AdlsLoggerTarget
                {
                    DebugMessageQueue = cmdlet.DebugMessages
                };
                // Add the target to the configuration
                _adlsLoggerConfig.AddTarget("logger", adlsTarget);

                //Logs all patterns of debug messages
                var rule = new LoggingRule("adls.dotnet.*", NLog.LogLevel.Debug, adlsTarget);
                _adlsLoggerConfig.LoggingRules.Add(rule);

               var powershellLoggingRule =
                    new LoggingRule("adls.powershell.WebTransport", NLog.LogLevel.Debug, adlsTarget);
                _adlsLoggerConfig.LoggingRules.Add(powershellLoggingRule);<commit_msg>Fix object reference null exception for diagnostic logging for bulkcopy<commit_after>﻿// ----------------------------------------------------------------------------------
//
// Copyright Microsoft Corporation
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ----------------------------------------------------------------------------------

using Microsoft.Azure.Commands.Common.Authentication.Abstractions;
using Microsoft.Azure.Commands.DataLakeStore.Properties;
using Microsoft.WindowsAzure.Commands.Utilities.Common;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Management.Automation;
using System.Net;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Azure.DataLake.Store;
using Microsoft.Azure.DataLake.Store.Acl;
using Microsoft.Azure.DataLake.Store.AclTools;
using Microsoft.Azure.DataLake.Store.FileTransfer;
using Microsoft.Azure.DataLake.Store.MockAdlsFileSystem;
using NLog;
using NLog.Config;
using NLog.Targets;
namespace Microsoft.Azure.Commands.DataLakeStore.Models
{
    public class DataLakeStoreFileSystemClient
    {

        /// <summary>
        /// The lock object
        /// </summary>
        private static readonly object ConsoleOutputLock = new object();

        private readonly IAzureContext _context;
        private readonly Random _uniqueActivityIdGenerator;
        private const int MaxConnectionLimit = 1000;
        private const long NeverExpireValue = 253402300800000;
        internal const int ImportExportMaxThreads = 256;
        private readonly LoggingConfiguration _adlsLoggerConfig;
        private readonly bool _isDebugEnabled;
        private const int DebugMessageFlushThreshold = 500;
        #region Constructors

        static DataLakeStoreFileSystemClient()
        {
            // Registering the custom target class
            Target.Register<AdlsLoggerTarget>("AdlsLogger"); //generic
            LogManager.ReconfigExistingLoggers();
        }

        public DataLakeStoreFileSystemClient(IAzureContext context)
        {
            if (context != null)
            {
                _context = context;
            }
            else
            {
                throw new ApplicationException(Resources.InvalidDefaultSubscription);
            }
            _uniqueActivityIdGenerator = new Random();

            // We need to override the default .NET value for max connections to a host to our number of threads, if necessary.
            // Otherwise we won't achieve the parallelism we want.
            // This is also required before the first call on the data lake store client.
            ServicePointManager.DefaultConnectionLimit = Math.Max(MaxConnectionLimit,
                ServicePointManager.DefaultConnectionLimit);
            ServicePointManager.Expect100Continue = false;
            ServicePointManager.UseNagleAlgorithm = false;
        }

        public DataLakeStoreFileSystemClient(IAzureContext context, DataLakeStoreFileSystemCmdletBase cmdlet) : this(context)
        {
            bool containsDebug = cmdlet.MyInvocation.BoundParameters.ContainsKey("Debug");
            if (containsDebug)
            {
                _isDebugEnabled = ((SwitchParameter) cmdlet.MyInvocation.BoundParameters["Debug"]).ToBool();
            }
            else
            {
                var debugPreference = cmdlet.GetVariableValue("DebugPreference") as string;
                if (debugPreference != null)
                {
                    _isDebugEnabled = !debugPreference.Equals("SilentlyContinue");
                }
            }

            // Keep this outside if block because it is also used for diagnostic file loggind for BulkCopy
            _adlsLoggerConfig = new LoggingConfiguration();
            if (_isDebugEnabled)
            {
                // Custom target that logs the debug messages from the SDK to the powershell framework's debug message queue
                var adlsTarget = new AdlsLoggerTarget
                {
                    DebugMessageQueue = cmdlet.DebugMessages
                };
                // Add the target to the configuration
                _adlsLoggerConfig.AddTarget("logger", adlsTarget);

                //Logs all patterns of debug messages
                var rule = new LoggingRule("adls.dotnet.*", NLog.LogLevel.Debug, adlsTarget);
                _adlsLoggerConfig.LoggingRules.Add(rule);

               var powershellLoggingRule =
                    new LoggingRule("adls.powershell.WebTransport", NLog.LogLevel.Debug, adlsTarget);
                _adlsLoggerConfig.LoggingRules.Add(powershellLoggingRule);
{'fraction_non_alphanumeric': 0.10004914004914005, 'fraction_numerical': 0.00628992628992629, 'mean_word_length': 2.1849765258215963, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4745955', 'n_tokens_mistral': 2414, 'n_tokens_neox': 2311, 'n_words': 690}