<commit_before>﻿using NLog;
using System;
using System.Collections.Generic;
using PaletteInsightAgent.Output;
using System.IO;

namespace PaletteInsightAgent.LogPoller
{

    class LogPollerAgent
    {
        private static readonly Logger Log = LogManager.GetCurrentClassLogger();
        public static readonly string InProgressLock = "Log Poller";

        private ICollection<LogFileWatcher> watchers;
        private LogsToDbConverter logsToDbConverter;

        /// <summary>
        /// The maximum log lines in a batch for ChangeDelegate.
        /// </summary>
        private int logLinesPerBatch;


        private ICollection<PaletteInsightAgentOptions.LogFolderInfo> foldersToWatch;


        public LogPollerAgent(ICollection<PaletteInsightAgentOptions.LogFolderInfo> foldersToWatch, int logLinesPerBatch)
        {
            Log.Info("Initializing LogPollerAgent with number of folders: {0}.", foldersToWatch.Count);
            this.foldersToWatch = foldersToWatch;
            this.logLinesPerBatch = logLinesPerBatch;
            logsToDbConverter = new LogsToDbConverter();
        }

        // Start the log file watchers
        public void start()
        {

            watchers = new List<LogFileWatcher>();
            foreach (var folderInfo in foldersToWatch)
            {
                Log.Info("Starting LogFileWatcher in " + folderInfo.FolderToWatch + " with file mask:" + folderInfo.DirectoryFilter);
                watchers.Add(new LogFileWatcher(folderInfo.FolderToWatch, folderInfo.DirectoryFilter, logLinesPerBatch, folderInfo.LogFormat));
            }
        }

        // Stop the log file watchers
        public void stop()
        {
            Log.Info("Stopping LogFileWatcher.");
        }


        public void pollLogs()
        {
            // Keep track of the files we have already processed, so we wont have
            // doubles if a file matches multiple watchers
            var alreadySubmittedFiles = new HashSet<string>();

            foreach (var watcher in watchers)
            {
                watcher.watchChangeCycle((filename, lines) =>
                {
                    // make sure we are using absolute paths for the checks
                    var fullPath = Path.GetFullPath(filename);

                    // check if we have already processed this file, and skip processing it it we have
                    if (alreadySubmittedFiles.Contains(fullPath))
                    {
                        Log.Info("Skippping already processed change in '{0}'", fullPath);
                        return;

                    }<commit_msg>fix: log skipping double-uploads as WARN<commit_after>﻿using NLog;
using System;
using System.Collections.Generic;
using PaletteInsightAgent.Output;
using System.IO;

namespace PaletteInsightAgent.LogPoller
{

    class LogPollerAgent
    {
        private static readonly Logger Log = LogManager.GetCurrentClassLogger();
        public static readonly string InProgressLock = "Log Poller";

        private ICollection<LogFileWatcher> watchers;
        private LogsToDbConverter logsToDbConverter;

        /// <summary>
        /// The maximum log lines in a batch for ChangeDelegate.
        /// </summary>
        private int logLinesPerBatch;


        private ICollection<PaletteInsightAgentOptions.LogFolderInfo> foldersToWatch;


        public LogPollerAgent(ICollection<PaletteInsightAgentOptions.LogFolderInfo> foldersToWatch, int logLinesPerBatch)
        {
            Log.Info("Initializing LogPollerAgent with number of folders: {0}.", foldersToWatch.Count);
            this.foldersToWatch = foldersToWatch;
            this.logLinesPerBatch = logLinesPerBatch;
            logsToDbConverter = new LogsToDbConverter();
        }

        // Start the log file watchers
        public void start()
        {

            watchers = new List<LogFileWatcher>();
            foreach (var folderInfo in foldersToWatch)
            {
                Log.Info("Starting LogFileWatcher in " + folderInfo.FolderToWatch + " with file mask:" + folderInfo.DirectoryFilter);
                watchers.Add(new LogFileWatcher(folderInfo.FolderToWatch, folderInfo.DirectoryFilter, logLinesPerBatch, folderInfo.LogFormat));
            }
        }

        // Stop the log file watchers
        public void stop()
        {
            Log.Info("Stopping LogFileWatcher.");
        }


        public void pollLogs()
        {
            // Keep track of the files we have already processed, so we wont have
            // doubles if a file matches multiple watchers
            var alreadySubmittedFiles = new HashSet<string>();

            foreach (var watcher in watchers)
            {
                watcher.watchChangeCycle((filename, lines) =>
                {
                    // make sure we are using absolute paths for the checks
                    var fullPath = Path.GetFullPath(filename);

                    // check if we have already processed this file, and skip processing it it we have
                    if (alreadySubmittedFiles.Contains(fullPath))
                    {
                        Log.Warn("Skippping already processed change in '{0}'", fullPath);
                        return;

                    }
{'fraction_non_alphanumeric': 0.07179487179487179, 'fraction_numerical': 0.0007597340930674264, 'mean_word_length': 2.024698449167145, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6194156', 'n_tokens_mistral': 1324, 'n_tokens_neox': 1238, 'n_words': 367}