<commit_before>﻿// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the MIT License. See License.txt in the project root for license information.

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Web.Http;
using Microsoft.Azure.WebJobs.Host;
using Microsoft.WindowsAzure.Storage.Blob;
using Microsoft.WindowsAzure.Storage.Queue;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Xunit;
using Microsoft.WebJobs.Script.Tests;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Azure.WebJobs.Script.Binding;

namespace Microsoft.Azure.WebJobs.Script.Tests
{
    [Trait("Category", "E2E")]
    [Trait("E2E", nameof(NodeEndToEndTests))]
    public class NodeEndToEndTests : EndToEndTestsBase<NodeEndToEndTests.TestFixture>
    {
        public NodeEndToEndTests(TestFixture fixture) : base(fixture)
        {
        }

        [Fact]
        public async Task BlobTriggerToBlobTest()
        {
            TestHelpers.ClearFunctionLogs("BlobTriggerToBlob");

            // write a binary blob
            string name = Guid.NewGuid().ToString();
            CloudBlockBlob inputBlob = Fixture.TestInputContainer.GetBlockBlobReference(name);
            inputBlob.Metadata.Add("TestMetadataKey", "TestMetadataValue");
            byte[] inputBytes = new byte[] { 1, 2, 3, 4, 5 };
            using (var stream = await inputBlob.OpenWriteAsync())
            {
                stream.Write(inputBytes, 0, inputBytes.Length);
            }

            var resultBlob = Fixture.TestOutputContainer.GetBlockBlobReference(name);
            await TestHelpers.WaitForBlobAsync(resultBlob);

            byte[] resultBytes;
            using (var resultStream = await resultBlob.OpenReadAsync())
            using (var ms = new MemoryStream())
            {
                resultStream.CopyTo(ms);
                resultBytes = ms.ToArray();
            }

            JObject testResult = await GetFunctionTestResult("BlobTriggerToBlob");
            Assert.Equal(inputBytes, resultBytes);
            Assert.True((bool)testResult["isBuffer"]);
            Assert.Equal(5, (int)testResult["length"]);

            var blobMetadata = (JObject)testResult["blobMetadata"];
            Assert.Equal($"test-input-node/{name}", (string)blobMetadata["path"]);

            var metadata = (JObject)blobMetadata["metadata"];
            Assert.Equal("TestMetadataValue", (string)metadata["TestMetadataKey"]);

            var properties = (JObject)blobMetadata["properties"];
            Assert.Equal("application/octet-stream", (string)properties["ContentType"]);
            Assert.Equal("BlockBlob", Enum.Parse(typeof(BlobType), (string)properties["BlobType"]).ToString());
            Assert.Equal(5, properties["Length"]);

            string invocationId = (string)testResult["invocationId"];
            Guid.Parse(invocationId);
        }

        [Fact]
        public async Task QueueTriggerByteArray()
        {
            TestHelpers.ClearFunctionLogs("QueueTriggerByteArray");

            // write a binary queue message
            byte[] inputBytes = new byte[] { 1, 2, 3 };
            CloudQueueMessage message = CloudQueueMessage.CreateCloudQueueMessageFromByteArray(inputBytes);
            var queue = Fixture.QueueClient.GetQueueReference("test-input-byte");
            await queue.CreateIfNotExistsAsync();
            await queue.ClearAsync();
            await queue.AddMessageAsync(message);

            JObject testResult = await GetFunctionTestResult("QueueTriggerByteArray");
            Assert.True((bool)testResult["isBuffer"]);
            Assert.Equal(5, (int)testResult["length"]);
        }

        /// <summary>
        /// Function "Invalid" has a binding error. This function validates that the error
        /// is cached, and the Fact that all the other tests in this suite run verifies that
        /// the error did not bring down the host.
        /// </summary>
        [Fact]
        public void ErrorFunction_DoesNotBringDownHost()
        {
            // verify the cached error for the invalid function
            string error = Fixture.Host.FunctionErrors["Invalid"].Single();
            Assert.Equal("'invalid' is not a valid binding direction.", error);
        }

        [Fact]
        public async Task ManualTrigger_Invoke_Succeeds()
        {
            await ManualTrigger_Invoke_SucceedsTest();
        }

        [Fact]
        public async Task QueueTriggerToBlob()
        {
            await QueueTriggerToBlobTest();
        }

        [Fact]
        public async Task Scenario_BindingData()
        {
            JObject input = new JObject
            {
                { "scenario", "bindingData" }
            };
            Dictionary<string, object> arguments = new Dictionary<string, object>
            {
                { "input", input.ToString() }
            };

            // assertions are done in the script itself, so if this function succeeds
            // we're good
            await Fixture.Host.CallAsync("Scenarios", arguments);
        }

        [Fact(Skip = "Issues with how log entry objecs are serialized (see https://github.com/Azure/azure-webjobs-sdk-script/issues/2020")]
        public async Task Scenario_Logging()
        {
            // Sleep to make sure all logs from other "Scenarios" tests have flushed before
            // we delete the file.
            await Task.Delay(1000);
            TestHelpers.ClearFunctionLogs("Scenarios");

            string testData = Guid.NewGuid().ToString();
            JObject input = new JObject
            {
                { "scenario", "logging" },
                { "input", testData },
            };
            Dictionary<string, object> arguments = new Dictionary<string, object>
            {
                { "input", input.ToString() }
            };
            await Fixture.Host.CallAsync("Scenarios", arguments);

            IList<string> logs = null;
            await TestHelpers.Await(() =>
            {
                logs = TestHelpers.GetFunctionLogsAsync("Scenarios", throwOnNoLogs: false).Result;
                return logs.Count > 0;
            });

            // verify use of context.log to log complex objects
            TraceEvent scriptTrace = Fixture.TraceWriter.Traces.Single(p => p.Message.Contains(testData));
            Assert.Equal(System.Diagnostics.TraceLevel.Info, scriptTrace.Level);
            JObject logEntry = JObject.Parse(scriptTrace.Message);
            Assert.Equal("This is a test", logEntry["message"]);
            Assert.Equal(testData, logEntry["input"]);

            // verify log levels in traces
            TraceEvent[] traces = Fixture.TraceWriter.Traces.Where(t => t.Message.Contains("loglevel")).ToArray();
            Assert.Equal(System.Diagnostics.TraceLevel.Info, traces[0].Level);
            Assert.Equal("loglevel default", traces[0].Message);
            Assert.Equal(System.Diagnostics.TraceLevel.Info, traces[1].Level);
            Assert.Equal("loglevel info", traces[1].Message);
            Assert.Equal(System.Diagnostics.TraceLevel.Verbose, traces[2].Level);
            Assert.Equal("loglevel verbose", traces[2].Message);
            Assert.Equal(System.Diagnostics.TraceLevel.Warning, traces[3].Level);
            Assert.Equal("loglevel warn", traces[3].Message);
            Assert.Equal(System.Diagnostics.TraceLevel.Error, traces[4].Level);
            Assert.Equal("loglevel error", traces[4].Message);

            // verify most of the logs look correct
            Assert.EndsWith("<NAME>", logs[3]);
            Assert.EndsWith("null", logs[4]);
            Assert.EndsWith("1234", logs[5]);
            Assert.EndsWith("true", logs[6]);
            Assert.EndsWith("loglevel default", logs[7]);
            Assert.EndsWith("loglevel info", logs[8]);
        }

        private async Task<CloudBlobContainer> GetEmptyContainer(string containerName)
        {
            var container = Fixture.BlobClient.GetContainerReference(containerName);
            if (await container.ExistsAsync())
            {
                foreach (CloudBlockBlob blob in (await container.ListBlobsSegmentedAsync(new BlobContinuationToken())).Results)
                {
                    await blob.DeleteAsync();
                }
            }
            return container;
        }

        [Fact]
        public async Task Scenario_RandGuidBinding_GeneratesRandomIDs()
        {
            var container = await GetEmptyContainer("scenarios-output");

            // Call 3 times - expect 3 separate output blobs
            for (int i = 0; i < 3; i++)
            {
                JObject input = new JObject
                {<commit_msg>Bump index numbers back to v1 logging format<commit_after>﻿// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the MIT License. See License.txt in the project root for license information.

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Web.Http;
using Microsoft.Azure.WebJobs.Host;
using Microsoft.WindowsAzure.Storage.Blob;
using Microsoft.WindowsAzure.Storage.Queue;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Xunit;
using Microsoft.WebJobs.Script.Tests;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Azure.WebJobs.Script.Binding;

namespace Microsoft.Azure.WebJobs.Script.Tests
{
    [Trait("Category", "E2E")]
    [Trait("E2E", nameof(NodeEndToEndTests))]
    public class NodeEndToEndTests : EndToEndTestsBase<NodeEndToEndTests.TestFixture>
    {
        public NodeEndToEndTests(TestFixture fixture) : base(fixture)
        {
        }

        [Fact]
        public async Task BlobTriggerToBlobTest()
        {
            TestHelpers.ClearFunctionLogs("BlobTriggerToBlob");

            // write a binary blob
            string name = Guid.NewGuid().ToString();
            CloudBlockBlob inputBlob = Fixture.TestInputContainer.GetBlockBlobReference(name);
            inputBlob.Metadata.Add("TestMetadataKey", "TestMetadataValue");
            byte[] inputBytes = new byte[] { 1, 2, 3, 4, 5 };
            using (var stream = await inputBlob.OpenWriteAsync())
            {
                stream.Write(inputBytes, 0, inputBytes.Length);
            }

            var resultBlob = Fixture.TestOutputContainer.GetBlockBlobReference(name);
            await TestHelpers.WaitForBlobAsync(resultBlob);

            byte[] resultBytes;
            using (var resultStream = await resultBlob.OpenReadAsync())
            using (var ms = new MemoryStream())
            {
                resultStream.CopyTo(ms);
                resultBytes = ms.ToArray();
            }

            JObject testResult = await GetFunctionTestResult("BlobTriggerToBlob");
            Assert.Equal(inputBytes, resultBytes);
            Assert.True((bool)testResult["isBuffer"]);
            Assert.Equal(5, (int)testResult["length"]);

            var blobMetadata = (JObject)testResult["blobMetadata"];
            Assert.Equal($"test-input-node/{name}", (string)blobMetadata["path"]);

            var metadata = (JObject)blobMetadata["metadata"];
            Assert.Equal("TestMetadataValue", (string)metadata["TestMetadataKey"]);

            var properties = (JObject)blobMetadata["properties"];
            Assert.Equal("application/octet-stream", (string)properties["ContentType"]);
            Assert.Equal("BlockBlob", Enum.Parse(typeof(BlobType), (string)properties["BlobType"]).ToString());
            Assert.Equal(5, properties["Length"]);

            string invocationId = (string)testResult["invocationId"];
            Guid.Parse(invocationId);
        }

        [Fact]
        public async Task QueueTriggerByteArray()
        {
            TestHelpers.ClearFunctionLogs("QueueTriggerByteArray");

            // write a binary queue message
            byte[] inputBytes = new byte[] { 1, 2, 3 };
            CloudQueueMessage message = CloudQueueMessage.CreateCloudQueueMessageFromByteArray(inputBytes);
            var queue = Fixture.QueueClient.GetQueueReference("test-input-byte");
            await queue.CreateIfNotExistsAsync();
            await queue.ClearAsync();
            await queue.AddMessageAsync(message);

            JObject testResult = await GetFunctionTestResult("QueueTriggerByteArray");
            Assert.True((bool)testResult["isBuffer"]);
            Assert.Equal(5, (int)testResult["length"]);
        }

        /// <summary>
        /// Function "Invalid" has a binding error. This function validates that the error
        /// is cached, and the Fact that all the other tests in this suite run verifies that
        /// the error did not bring down the host.
        /// </summary>
        [Fact]
        public void ErrorFunction_DoesNotBringDownHost()
        {
            // verify the cached error for the invalid function
            string error = Fixture.Host.FunctionErrors["Invalid"].Single();
            Assert.Equal("'invalid' is not a valid binding direction.", error);
        }

        [Fact]
        public async Task ManualTrigger_Invoke_Succeeds()
        {
            await ManualTrigger_Invoke_SucceedsTest();
        }

        [Fact]
        public async Task QueueTriggerToBlob()
        {
            await QueueTriggerToBlobTest();
        }

        [Fact]
        public async Task Scenario_BindingData()
        {
            JObject input = new JObject
            {
                { "scenario", "bindingData" }
            };
            Dictionary<string, object> arguments = new Dictionary<string, object>
            {
                { "input", input.ToString() }
            };

            // assertions are done in the script itself, so if this function succeeds
            // we're good
            await Fixture.Host.CallAsync("Scenarios", arguments);
        }

        [Fact]
        public async Task Scenario_Logging()
        {
            // Sleep to make sure all logs from other "Scenarios" tests have flushed before
            // we delete the file.
            await Task.Delay(1000);
            TestHelpers.ClearFunctionLogs("Scenarios");

            string testData = Guid.NewGuid().ToString();
            JObject input = new JObject
            {
                { "scenario", "logging" },
                { "input", testData },
            };
            Dictionary<string, object> arguments = new Dictionary<string, object>
            {
                { "input", input.ToString() }
            };
            await Fixture.Host.CallAsync("Scenarios", arguments);

            IList<string> logs = null;
            await TestHelpers.Await(() =>
            {
                logs = TestHelpers.GetFunctionLogsAsync("Scenarios", throwOnNoLogs: false).Result;
                return logs.Count > 0;
            });

            // verify use of context.log to log complex objects
            TraceEvent scriptTrace = Fixture.TraceWriter.Traces.Single(p => p.Message.Contains(testData));
            Assert.Equal(System.Diagnostics.TraceLevel.Info, scriptTrace.Level);
            JObject logEntry = JObject.Parse(scriptTrace.Message);
            Assert.Equal("This is a test", logEntry["message"]);
            Assert.Equal(testData, logEntry["input"]);

            // verify log levels in traces
            TraceEvent[] traces = Fixture.TraceWriter.Traces.Where(t => t.Message.Contains("loglevel")).ToArray();
            Assert.Equal(System.Diagnostics.TraceLevel.Info, traces[0].Level);
            Assert.Equal("loglevel default", traces[0].Message);
            Assert.Equal(System.Diagnostics.TraceLevel.Info, traces[1].Level);
            Assert.Equal("loglevel info", traces[1].Message);
            Assert.Equal(System.Diagnostics.TraceLevel.Verbose, traces[2].Level);
            Assert.Equal("loglevel verbose", traces[2].Message);
            Assert.Equal(System.Diagnostics.TraceLevel.Warning, traces[3].Level);
            Assert.Equal("loglevel warn", traces[3].Message);
            Assert.Equal(System.Diagnostics.TraceLevel.Error, traces[4].Level);
            Assert.Equal("loglevel error", traces[4].Message);

            // verify most of the logs look correct
            Assert.EndsWith("<NAME>", logs[4]);
            Assert.EndsWith("null", logs[5]);
            Assert.EndsWith("1234", logs[6]);
            Assert.EndsWith("true", logs[7]);
            Assert.EndsWith("loglevel default", logs[8]);
            Assert.EndsWith("loglevel info", logs[9]);
        }

        private async Task<CloudBlobContainer> GetEmptyContainer(string containerName)
        {
            var container = Fixture.BlobClient.GetContainerReference(containerName);
            if (await container.ExistsAsync())
            {
                foreach (CloudBlockBlob blob in (await container.ListBlobsSegmentedAsync(new BlobContinuationToken())).Results)
                {
                    await blob.DeleteAsync();
                }
            }
            return container;
        }

        [Fact]
        public async Task Scenario_RandGuidBinding_GeneratesRandomIDs()
        {
            var container = await GetEmptyContainer("scenarios-output");

            // Call 3 times - expect 3 separate output blobs
            for (int i = 0; i < 3; i++)
            {
                JObject input = new JObject
                {
{'fraction_non_alphanumeric': 0.10850822461712989, 'fraction_numerical': 0.005161656267725468, 'mean_word_length': 2.213231273920175, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 31, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '978473', 'n_tokens_mistral': 4523, 'n_tokens_neox': 4444, 'n_words': 1009}