<commit_before>﻿namespace GleanETL.Test
{
    using System;
    using System.Data;
    using System.Data.SqlClient;
    using System.Diagnostics;
    using System.IO;
    using System.Text;
    using GleanETL.Core;
    using GleanETL.Core.Columns;
    using GleanETL.Core.Enumerations;
    using GleanETL.Core.Extraction;
    using GleanETL.Core.Source;
    using GleanETL.Core.Target;
    using Microsoft.VisualStudio.TestTools.UnitTesting;

    [TestClass]
    public class Scratch
    {
        #region Fields

        //private const string LocalDbConnectionString = @"Server=(localdb)\v12.0;Integrated Security=true;Initial Catalog=GleanETL;";
        private const string LocalDbConnectionString = @"Server=(local)\SQL2014;User ID=sa;Password=<PASSWORD>!;Initial Catalog=GleanETL;";

        #endregion Fields

        #region Methods

        [TestMethod]
        public void ExtractHostsFileToDatabase()
        {
            const string windowsUpdateLogFilePath = @"C:\Windows\System32\drivers\etc\HOSTS";

            if (!File.Exists(windowsUpdateLogFilePath))
            {
                Assert.Fail("The Windows Update log file does not exist!");
            }
            else
            {
                var source = new TextFileSource(windowsUpdateLogFilePath)
                {
                    TakeLineIf = line =>
                        line.Length > 0 &&
                        (!line.Contains("#") || line.IndexOf('#') > 0)
                };

                var target = new DatabaseTableTarget(LocalDbConnectionString, "Hosts", deleteTableIfExists: true);

                var columns = new[]
                {
                    new StringColumn("IpAddress"),
                    new StringColumn("Hostname", stringCapitalisation: StringCapitalisation.ToLowerCase)
                };

                var extraction = new LineExtraction<DatabaseTableTarget>(columns, source, target, throwParseErrors: true)
                {
                    SplitLineFunc =
                        line =>
                            line.OriginalLine.TrimAndRemoveConsecutiveWhitespace()
                                .Split(new[] { ' ' }, 2, StringSplitOptions.RemoveEmptyEntries)
                                .ForEachAssign((i, s) => (i > 0 && s.Contains("#")) ? s.Substring(0, s.IndexOf('#')) : s)
                };

                extraction.DataParseError += DataParseError;
                extraction.ExtractComplete += ExtractComplete;
                extraction.ExtractToTarget();
            }
        }

        [TestMethod]
        public void ExtractHostsFileToTraceOutput()
        {
            var columns = new[]
            {
                new StringColumn("IP", 16),
                new StringColumn("HOST", 250, stringCapitalisation: Core.Enumerations.StringCapitalisation.ToLowerCase)
            };

            var source = new TextFileSource(@"C:\Windows\System32\drivers\etc\HOSTS")
            {
                TakeLineIf = line =>
                    line.Length > 0 &&
                    (!line.Contains("#") || line.IndexOf('#') > 0)
            };

            var target = new TraceOutputTarget();

            var extraction = new LineExtraction<TraceOutputTarget>(columns, source, target, throwParseErrors: true)
            {
                SplitLineFunc = line =>
                        line.OriginalLine.TrimAndRemoveConsecutiveWhitespace()
                            .Split(new[] { ' ' }, 2, StringSplitOptions.RemoveEmptyEntries)
                            .ForEachAssign((i, s) => (i > 0 && s.Contains("#")) ? s.Substring(0, s.IndexOf('#')) : s)
            };

            extraction.DataParseError += DataParseError;
            extraction.ExtractComplete += ExtractComplete;
            extraction.ExtractToTarget();
        }

        [TestMethod]
        public void ExtractWindowsUpdateLogFileToDatabase()
        {
            const string windowsUpdateLog = @"C:\Windows\WindowsUpdate.log";

            if (File.Exists(windowsUpdateLog))
            {
                string sourceFile = Path.GetTempFileName();
                File.Copy(windowsUpdateLog, sourceFile, true);

                var columns = new BaseColumn[]
                {
                    new DateColumn("Timestamp", new[] {"yyyy-MM-dd HH:mm:ss:fff"}, "yyyy-MM-dd HH:mm:ss.fff"),
                    new IgnoredColumn(),
                    new IgnoredColumn(),
                    new StringColumn("Message")
                };

                var source = new TextFileSource(sourceFile)
                {
                    TakeLineIf = line => line.Length > 0
                };

                var target = new DatabaseTableTarget(LocalDbConnectionString, "WindowsUpdateLog", deleteTableIfExists: true);

                var extraction = new LineExtraction<DatabaseTableTarget>(columns, source, target, throwParseErrors: true)
                {
                    SplitLineFunc = line => line.Split(0, 23, 28, 33)
                };

                extraction.DataParseError += DataParseError;
                extraction.ExtractComplete += ExtractComplete;
                extraction.ExtractToTarget();

                if (File.Exists(sourceFile))
                {
                    File.Delete(sourceFile);
                }
            }
            else
            {
                Assert.Inconclusive("File could not be found: {0}.", windowsUpdateLog);
            }
        }

        [TestInitialize]
        public void TestInit()
        {
            var csb = new SqlConnectionStringBuilder(LocalDbConnectionString);
            csb.InitialCatalog = "master";

            using (var c = new SqlConnection(csb.ConnectionString))
            {
                using (var cmd = c.CreateCommand())
                {
                    cmd.CommandType = CommandType.Text;
                    cmd.CommandText =
                        @"
            IF NOT EXISTS (
            SELECT 1
            FROM sys.databases
            WHERE [name] = 'GleanETL'
            )
            CREATE DATABASE [GleanETL];
            ";

                    c.Open();
                    cmd.ExecuteNonQuery();
                }<commit_msg>database connection string tweak for appveyor<commit_after>﻿namespace GleanETL.Test
{
    using System;
    using System.Data;
    using System.Data.SqlClient;
    using System.Diagnostics;
    using System.IO;
    using System.Text;
    using GleanETL.Core;
    using GleanETL.Core.Columns;
    using GleanETL.Core.Enumerations;
    using GleanETL.Core.Extraction;
    using GleanETL.Core.Source;
    using GleanETL.Core.Target;
    using Microsoft.VisualStudio.TestTools.UnitTesting;

    [TestClass]
    public class Scratch
    {
        #region Fields

        private const string LocalDbConnectionString = @"Server=(localDB)\MSSQLLocalDB;Integrated Security=true;Initial Catalog=GleanETL;";
        private const string AppVeyorDbConnectionString = @"Server=(local)\SQL2014;User ID=sa;Password=<PASSWORD>!;Initial Catalog=GleanETL;";
        private static string _databaseConnectionString = LocalDbConnectionString;

        #endregion Fields

        #region Methods

        [ClassInitialize]
        public static void ClassInitialize(TestContext ctx)
        {
            // this is a crude way of detecting if the tests are running in an AppVeyor build.
            if (Directory.Exists(@"C:\Program Files\AppVeyor") || Directory.Exists(@"C:\Program Files (x86)\AppVeyor"))
            {
                _databaseConnectionString = AppVeyorDbConnectionString;
            }
        }


        [TestMethod]
        public void ExtractHostsFileToDatabase()
        {
            const string windowsUpdateLogFilePath = @"C:\Windows\System32\drivers\etc\HOSTS";

            if (!File.Exists(windowsUpdateLogFilePath))
            {
                Assert.Fail("The Windows Update log file does not exist!");
            }
            else
            {
                var source = new TextFileSource(windowsUpdateLogFilePath)
                {
                    TakeLineIf = line =>
                        line.Length > 0 &&
                        (!line.Contains("#") || line.IndexOf('#') > 0)
                };

                var target = new DatabaseTableTarget(_databaseConnectionString, "Hosts", deleteTableIfExists: true);

                var columns = new[]
                {
                    new StringColumn("IpAddress"),
                    new StringColumn("Hostname", stringCapitalisation: StringCapitalisation.ToLowerCase)
                };

                var extraction = new LineExtraction<DatabaseTableTarget>(columns, source, target, throwParseErrors: true)
                {
                    SplitLineFunc =
                        line =>
                            line.OriginalLine.TrimAndRemoveConsecutiveWhitespace()
                                .Split(new[] { ' ' }, 2, StringSplitOptions.RemoveEmptyEntries)
                                .ForEachAssign((i, s) => (i > 0 && s.Contains("#")) ? s.Substring(0, s.IndexOf('#')) : s)
                };

                extraction.DataParseError += DataParseError;
                extraction.ExtractComplete += ExtractComplete;
                extraction.ExtractToTarget();
            }
        }

        [TestMethod]
        public void ExtractHostsFileToTraceOutput()
        {
            var columns = new[]
            {
                new StringColumn("IP", 16),
                new StringColumn("HOST", 250, stringCapitalisation: Core.Enumerations.StringCapitalisation.ToLowerCase)
            };

            var source = new TextFileSource(@"C:\Windows\System32\drivers\etc\HOSTS")
            {
                TakeLineIf = line =>
                    line.Length > 0 &&
                    (!line.Contains("#") || line.IndexOf('#') > 0)
            };

            var target = new TraceOutputTarget();

            var extraction = new LineExtraction<TraceOutputTarget>(columns, source, target, throwParseErrors: true)
            {
                SplitLineFunc = line =>
                        line.OriginalLine.TrimAndRemoveConsecutiveWhitespace()
                            .Split(new[] { ' ' }, 2, StringSplitOptions.RemoveEmptyEntries)
                            .ForEachAssign((i, s) => (i > 0 && s.Contains("#")) ? s.Substring(0, s.IndexOf('#')) : s)
            };

            extraction.DataParseError += DataParseError;
            extraction.ExtractComplete += ExtractComplete;
            extraction.ExtractToTarget();
        }

        [TestMethod]
        public void ExtractWindowsUpdateLogFileToDatabase()
        {
            const string windowsUpdateLog = @"C:\Windows\WindowsUpdate.log";

            if (File.Exists(windowsUpdateLog))
            {
                string sourceFile = Path.GetTempFileName();
                File.Copy(windowsUpdateLog, sourceFile, true);

                var columns = new BaseColumn[]
                {
                    new DateColumn("Timestamp", new[] {"yyyy-MM-dd HH:mm:ss:fff"}, "yyyy-MM-dd HH:mm:ss.fff"),
                    new IgnoredColumn(),
                    new IgnoredColumn(),
                    new StringColumn("Message")
                };

                var source = new TextFileSource(sourceFile)
                {
                    TakeLineIf = line => line.Length > 0
                };

                var target = new DatabaseTableTarget(_databaseConnectionString, "WindowsUpdateLog", deleteTableIfExists: true);

                var extraction = new LineExtraction<DatabaseTableTarget>(columns, source, target, throwParseErrors: true)
                {
                    SplitLineFunc = line => line.Split(0, 23, 28, 33)
                };

                extraction.DataParseError += DataParseError;
                extraction.ExtractComplete += ExtractComplete;
                extraction.ExtractToTarget();

                if (File.Exists(sourceFile))
                {
                    File.Delete(sourceFile);
                }
            }
            else
            {
                Assert.Inconclusive("File could not be found: {0}.", windowsUpdateLog);
            }
        }

        [TestInitialize]
        public void TestInit()
        {
            var csb = new SqlConnectionStringBuilder(_databaseConnectionString);
            csb.InitialCatalog = "master";

            using (var c = new SqlConnection(csb.ConnectionString))
            {
                using (var cmd = c.CreateCommand())
                {
                    cmd.CommandType = CommandType.Text;
                    cmd.CommandText =
                        @"
            IF NOT EXISTS (
            SELECT 1
            FROM sys.databases
            WHERE [name] = 'GleanETL'
            )
            CREATE DATABASE [GleanETL];
            ";

                    c.Open();
                    cmd.ExecuteNonQuery();
                }
{'fraction_non_alphanumeric': 0.09563473700447324, 'fraction_numerical': 0.005475859941385161, 'mean_word_length': 1.5500491642084562, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1170723', 'n_tokens_mistral': 3368, 'n_tokens_neox': 3079, 'n_words': 634}