<commit_before>﻿using System;
using System.Collections.Generic;
using System.Data;
using System.Linq;
using System.Text;
using Sqlconsulting.DataCollector.Utils;
using System.IO;

namespace Sqlconsulting.DataCollector.ExtendedTSQLCollector
{
    public class Collector
    {
        public Boolean verbose { get; set; }
 

        public void CollectData(
                String SourceServerInstance,
                Guid CollectionSetUid,
                int ItemId
            )
        {

            CollectorLogger logger = new CollectorLogger(SourceServerInstance, CollectionSetUid, ItemId);

            DataTable collectedData = null;

            if (verbose) logger.logMessage("--------------------------------");
            if (verbose) logger.logMessage("      ExtendedTSQLCollector      ");
            if (verbose) logger.logMessage("--------------------------------");
            if (verbose) logger.logMessage("Copyright© sqlconsulting.it 2014");
            if (verbose) logger.logMessage("-");

            if (verbose) logger.logMessage("Loading configuration");
            //
            // Load Configuration
            //
            CollectorConfig cfg = CollectorUtils.GetCollectorConfig(SourceServerInstance, CollectionSetUid, ItemId);


            if (verbose) logger.logMessage("Entering collection items loop");
            foreach (CollectionItemConfig itm in cfg.collectionItems)
            {
                if (verbose) logger.logMessage("Processing item n. " + itm.Index);
                if (verbose) logger.logMessage("Processing query " + itm.Query);

                collectedData = null;

                String ts = DateTime.Now.ToString("yyyyMMddHHmmss");
                String collectorId = CollectorUtils.getCacheFilePrefix(SourceServerInstance, CollectionSetUid, ItemId); 

                String destFile = Path.Combine(cfg.CacheDirectory, collectorId + "_" + ts + ".cache");

                //
                // Iterate through the enabled databases
                //
                if (verbose) logger.logMessage("Entering databases loop");
                foreach (String currentDatabase in cfg.Databases)
                {

                    if (verbose) logger.logMessage("Processing database " + currentDatabase);
                    //
                    // Execute the query in the collection item
                    //

                    DataTable dt = CollectorUtils.GetDataTable(SourceServerInstance, currentDatabase, itm.Query);

                    //
                    // Add computed columns
                    //
                    if (dt.Columns.Contains("database_name"))
                    {
                        dt.Columns["database_name"].ColumnName = "__database_name";
                    }
                    DataColumn cl_db = new DataColumn("database_name", typeof(String));
                    cl_db.DefaultValue = currentDatabase;
                    dt.Columns.Add(cl_db);


                    if (dt.Columns.Contains("collection_time"))<commit_msg>Fixed bug with multiple queries in a collection item<commit_after>﻿using System;
using System.Collections.Generic;
using System.Data;
using System.Linq;
using System.Text;
using Sqlconsulting.DataCollector.Utils;
using System.IO;

namespace Sqlconsulting.DataCollector.ExtendedTSQLCollector
{
    public class Collector
    {
        public Boolean verbose { get; set; }
 

        public void CollectData(
                String SourceServerInstance,
                Guid CollectionSetUid,
                int ItemId
            )
        {

            CollectorLogger logger = new CollectorLogger(SourceServerInstance, CollectionSetUid, ItemId);

            DataTable collectedData = null;

            if (verbose) logger.logMessage("--------------------------------");
            if (verbose) logger.logMessage("      ExtendedTSQLCollector      ");
            if (verbose) logger.logMessage("--------------------------------");
            if (verbose) logger.logMessage("Copyright© sqlconsulting.it 2014");
            if (verbose) logger.logMessage("-");

            if (verbose) logger.logMessage("Loading configuration");
            //
            // Load Configuration
            //
            CollectorConfig cfg = CollectorUtils.GetCollectorConfig(SourceServerInstance, CollectionSetUid, ItemId);


            if (verbose) logger.logMessage("Entering collection items loop");
            foreach (CollectionItemConfig itm in cfg.collectionItems)
            {
                if (verbose) logger.logMessage("Processing item n. " + itm.Index);
                if (verbose) logger.logMessage("Processing query " + itm.Query);

                collectedData = null;

                String ts = DateTime.Now.ToString("yyyyMMddHHmmss");
                String collectorId = CollectorUtils.getCacheFilePrefix(SourceServerInstance, CollectionSetUid, ItemId) + "_" + itm.Index; 

                String destFile = Path.Combine(cfg.CacheDirectory, collectorId + "_" + ts + ".cache");

                //
                // Iterate through the enabled databases
                //
                if (verbose) logger.logMessage("Entering databases loop");
                foreach (String currentDatabase in cfg.Databases)
                {

                    if (verbose) logger.logMessage("Processing database " + currentDatabase);
                    //
                    // Execute the query in the collection item
                    //

                    DataTable dt = CollectorUtils.GetDataTable(SourceServerInstance, currentDatabase, itm.Query);

                    //
                    // Add computed columns
                    //
                    if (dt.Columns.Contains("database_name"))
                    {
                        dt.Columns["database_name"].ColumnName = "__database_name";
                    }
                    DataColumn cl_db = new DataColumn("database_name", typeof(String));
                    cl_db.DefaultValue = currentDatabase;
                    dt.Columns.Add(cl_db);


                    if (dt.Columns.Contains("collection_time"))
{'fraction_non_alphanumeric': 0.10641754670999187, 'fraction_numerical': 0.0012997562956945572, 'mean_word_length': 1.653448275862069, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3397958', 'n_tokens_mistral': 1470, 'n_tokens_neox': 1357, 'n_words': 324}