<commit_before>﻿using System;
using System.IO;
using System.Data;
using NLog;
using System.Globalization;
using System.Collections.Generic;
using System.Linq;

namespace PaletteInsightAgent.Output
{
    class OutputSerializer
    {
        public const string DATA_FOLDER = "data/";
        private const string FILENAME_DATETIME_FORMAT = "yyyy-MM-dd--HH-mm-ss";
        public const string IN_PROGRESS_FILE_POSTFIX = ".writing";
        public const string MAX_ID_PREFIX = "maxid";
        private static readonly Logger Log = LogManager.GetCurrentClassLogger();
        private static IWriter Writer = new CsvSerializer();
        public static string Extension
        {
            get
            {
                return Writer.Extension;
            }
        }

        public static void Write(DataTable table, string maxId = null)
        {
            // skip writing if no data
            var rowCount = table.Rows.Count;
            if (rowCount == 0)
            {
                Log.Debug("No data - skipping CSV write.");
                return;
            }

            // write out if any
            var dataFileName = GetDataFile(table.TableName);

            Log.Info("Trying to write: {0} -- files in directory:", dataFileName);
            foreach (var filename in Directory.EnumerateFiles(Path.GetDirectoryName(dataFileName)))
            {
                Log.Info("   - {0}  ", filename);
            }

            try
            {

                Writer.WriteDataFile(dataFileName, table);

                // remove any rows from the csv queue
                table.Rows.Clear();

                // write the maxid file if it is not null
                if (maxId != null)
                {
                    var maxIdFileName = dataFileName + MAX_ID_PREFIX;
                    File.AppendAllText(maxIdFileName, maxId);
                }
            }
            catch (Exception e)
            {
                Log.Error("Failed to write {0} table contents to CSV file '{1}'! Exception message: {2}", table.TableName, dataFileName, e.Message);
            }
        }

        public static void Delete(string tableName)<commit_msg>removed excess logging of existing files<commit_after>﻿using System;
using System.IO;
using System.Data;
using NLog;
using System.Globalization;
using System.Collections.Generic;
using System.Linq;

namespace PaletteInsightAgent.Output
{
    class OutputSerializer
    {
        public const string DATA_FOLDER = "data/";
        private const string FILENAME_DATETIME_FORMAT = "yyyy-MM-dd--HH-mm-ss";
        public const string IN_PROGRESS_FILE_POSTFIX = ".writing";
        public const string MAX_ID_PREFIX = "maxid";
        private static readonly Logger Log = LogManager.GetCurrentClassLogger();
        private static IWriter Writer = new CsvSerializer();
        public static string Extension
        {
            get
            {
                return Writer.Extension;
            }
        }

        public static void Write(DataTable table, string maxId = null)
        {
            // skip writing if no data
            var rowCount = table.Rows.Count;
            if (rowCount == 0)
            {
                Log.Debug("No data - skipping CSV write.");
                return;
            }

            // get the filename here (should not throw), so we can log on failiures
            var dataFileName = GetDataFile(table.TableName);

            try
            {

                Writer.WriteDataFile(dataFileName, table);

                // remove any rows from the csv queue
                table.Rows.Clear();

                // write the maxid file if it is not null
                if (maxId != null)
                {
                    var maxIdFileName = dataFileName + MAX_ID_PREFIX;
                    File.AppendAllText(maxIdFileName, maxId);
                }
            }
            catch (Exception e)
            {
                Log.Error("Failed to write {0} table contents to CSV file '{1}'! Exception message: {2}", table.TableName, dataFileName, e.Message);
            }
        }

        public static void Delete(string tableName)
{'fraction_non_alphanumeric': 0.08063354931605471, 'fraction_numerical': 0.0023998080153587713, 'mean_word_length': 1.6164469554300063, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1076253', 'n_tokens_mistral': 1067, 'n_tokens_neox': 1009, 'n_words': 313}