<commit_before>﻿#region Copyright Notice

//    Copyright 2011-2013 <NAME>
// 
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
// 
//        http://www.apache.org/licenses/LICENSE-2.0
// 
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

#endregion

#region Using Directives

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Windows;
using LumenWorks.Framework.IO.Csv;

#endregion

namespace LeftosCommonLibrary
{
    /// <summary>
    ///     Provides methods to convert from and to CSV data.
    /// </summary>
    public static class CSV
    {
        private const string Quote = "\"";
        private const string EscapedQuote = "\"\"";
        private static readonly char[] CharactersThatMustBeQuoted = {',', '"', '\n', ' '};

        private static readonly char ListSeparator = CultureInfo.CurrentCulture.TextInfo.ListSeparator.ToCharArray()[0];

        private static char detectSeparator(string path)
        {
            string s = File.ReadAllText(path);
            return DetectSeparator(new StringReader(s), Tools.SplitLinesToArray(s).Length, new[] {',', ';'});
        }

        /// <summary>
        ///     Converts CSV data from a file into a list of dictionaries.
        /// </summary>
        /// <param name="path">The path of the CSV file.</param>
        /// <param name="useCultureSeparator">
        ///     If <c>true</c>, the method uses the current culture's list separator.
        ///     If <c>false</c>, it tries to automatically detect it.
        /// </param>
        /// <returns>
        ///     A list of dictionaries, where each dictionary is a record, and the key-value pairs are the column header and
        ///     corresponding value.
        /// </returns>
        public static List<Dictionary<string, string>> DictionaryListFromCSVFile(string path, bool useCultureSeparator = false)
        {
            var cr = new CsvReader(new StreamReader(path), true, useCultureSeparator ? ListSeparator : detectSeparator(path));
            List<Dictionary<string, string>> dictList = dictionaryListFromCSV(cr);

            return dictList;
        }

        /// <summary>
        ///     Converts CSV data from a string into a list of dictionaries.
        /// </summary>
        /// <param name="text">The CSV data.</param>
        /// <param name="useCultureSeparator">
        ///     If <c>true</c>, the method uses the current culture's list separator.
        ///     If <c>false</c>, it tries to automatically detect it.
        /// </param>
        /// <returns>
        ///     A list of dictionaries, where each dictionary is a record, and the key-value pairs are the column header and
        ///     corresponding value.
        /// </returns>
        public static List<Dictionary<string, string>> DictionaryListFromCSVString(string text, bool useCultureSeparator = false)
        {
            var cr = new CsvReader(new StringReader(text), true,
                                   useCultureSeparator ? ListSeparator : DetectSeparator(new StringReader(text), 1, new[] {',', ';'}));
            List<Dictionary<string, string>> dictList = dictionaryListFromCSV(cr);

            return dictList;
        }

        /// <summary>
        ///     Converts CSV data from a file into a list of string arrays.
        /// </summary>
        /// <param name="path">The path to the file.</param>
        /// <param name="hasHeaders">
        ///     if set to <c>true</c>, the file is assumed to have headers in the first row.
        /// </param>
        /// <param name="useCultureSeparator">
        ///     If <c>true</c>, the method uses the current culture's list separator.
        ///     If <c>false</c>, it tries to automatically detect it.
        /// </param>
        /// <returns>A list of string arrays containing the CSV data.</returns>
        public static List<string[]> ArrayListFromCSVFile(string path, bool hasHeaders = true, bool useCultureSeparator = false)
        {
            var cr = new CsvReader(new StreamReader(path), hasHeaders, useCultureSeparator ? ListSeparator : detectSeparator(path));
            List<string[]> arrayList = arrayListFromCSV(cr);

            return arrayList;
        }

        /// <summary>
        ///     Converts CSV data from a string into a list of string arrays.
        /// </summary>
        /// <param name="text">The CSV data.</param>
        /// <param name="hasHeaders">
        ///     if set to <c>true</c>, the file is assumed to have headers in the first row.
        /// </param>
        /// <param name="useCultureSeparator">
        ///     If <c>true</c>, the method uses the current culture's list separator.
        ///     If <c>false</c>, it tries to automatically detect it.
        /// </param>
        /// <returns>A list of string arrays containing the CSV data.</returns>
        public static List<string[]> ArrayListFromCSVString(string text, bool hasHeaders = true, bool useCultureSeparator = false)
        {
            var cr = new CsvReader(new StringReader(text), hasHeaders,
                                   useCultureSeparator ? ListSeparator : DetectSeparator(new StringReader(text), 1, new[] {',', ';'}));
            List<string[]> arrayList = arrayListFromCSV(cr);

            return arrayList;
        }

        private static List<Dictionary<string, string>> dictionaryListFromCSV(CsvReader cr)
        {
            List<Dictionary<string, string>> dictList;
            using (cr)
            {
                dictList = new List<Dictionary<string, string>>();

                int fieldCount = cr.FieldCount;
                string[] headers = cr.GetFieldHeaders();

                for (int i = 0; i < headers.Length; i++)
                {
                    var sortingChars = new[] {'^', Convert.ToChar(65533)};
                    char firstChar = Convert.ToChar(headers[i].Substring(0, 1));
                    if (sortingChars.Contains(firstChar))
                    {
                        headers[i] = headers[i].Split(new[] {" ", "\r\n", "\n"}, 2, StringSplitOptions.None)[1];
                        break;
                    }
                }

                int j = 0;
                while (cr.ReadNextRecord())<commit_msg>Fixed # being a comment denoting character when parsing CSV data<commit_after>﻿#region Copyright Notice

//    Copyright 2011-2013 <NAME>
// 
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
// 
//        http://www.apache.org/licenses/LICENSE-2.0
// 
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

#endregion

#region Using Directives

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Windows;
using LumenWorks.Framework.IO.Csv;

#endregion

namespace LeftosCommonLibrary
{
    /// <summary>
    ///     Provides methods to convert from and to CSV data.
    /// </summary>
    public static class CSV
    {
        private const string Quote = "\"";
        private const string EscapedQuote = "\"\"";
        private static readonly char[] CharactersThatMustBeQuoted = {',', '"', '\n', ' '};

        private static readonly char ListSeparator = CultureInfo.CurrentCulture.TextInfo.ListSeparator.ToCharArray()[0];

        private static char detectSeparator(string path)
        {
            string s = File.ReadAllText(path);
            return DetectSeparator(new StringReader(s), Tools.SplitLinesToArray(s).Length, new[] {',', ';'});
        }

        /// <summary>
        ///     Converts CSV data from a file into a list of dictionaries.
        /// </summary>
        /// <param name="path">The path of the CSV file.</param>
        /// <param name="useCultureSeparator">
        ///     If <c>true</c>, the method uses the current culture's list separator.
        ///     If <c>false</c>, it tries to automatically detect it.
        /// </param>
        /// <returns>
        ///     A list of dictionaries, where each dictionary is a record, and the key-value pairs are the column header and
        ///     corresponding value.
        /// </returns>
        public static List<Dictionary<string, string>> DictionaryListFromCSVFile(string path, bool useCultureSeparator = false)
        {
            var cr = new CsvReader(new StreamReader(path), true, useCultureSeparator ? ListSeparator : detectSeparator(path), '\"',
                                   '\"', '\0', ValueTrimmingOptions.UnquotedOnly);
            List<Dictionary<string, string>> dictList = dictionaryListFromCSV(cr);

            return dictList;
        }

        /// <summary>
        ///     Converts CSV data from a string into a list of dictionaries.
        /// </summary>
        /// <param name="text">The CSV data.</param>
        /// <param name="useCultureSeparator">
        ///     If <c>true</c>, the method uses the current culture's list separator.
        ///     If <c>false</c>, it tries to automatically detect it.
        /// </param>
        /// <returns>
        ///     A list of dictionaries, where each dictionary is a record, and the key-value pairs are the column header and
        ///     corresponding value.
        /// </returns>
        public static List<Dictionary<string, string>> DictionaryListFromCSVString(string text, bool useCultureSeparator = false)
        {
            var cr = new CsvReader(new StringReader(text), true,
                                   useCultureSeparator ? ListSeparator : DetectSeparator(new StringReader(text), 1, new[] {',', ';'}),
                                   '\"', '\"', '\0', ValueTrimmingOptions.UnquotedOnly);
            List<Dictionary<string, string>> dictList = dictionaryListFromCSV(cr);

            return dictList;
        }

        /// <summary>
        ///     Converts CSV data from a file into a list of string arrays.
        /// </summary>
        /// <param name="path">The path to the file.</param>
        /// <param name="hasHeaders">
        ///     if set to <c>true</c>, the file is assumed to have headers in the first row.
        /// </param>
        /// <param name="useCultureSeparator">
        ///     If <c>true</c>, the method uses the current culture's list separator.
        ///     If <c>false</c>, it tries to automatically detect it.
        /// </param>
        /// <returns>A list of string arrays containing the CSV data.</returns>
        public static List<string[]> ArrayListFromCSVFile(string path, bool hasHeaders = true, bool useCultureSeparator = false)
        {
            var cr = new CsvReader(new StreamReader(path), hasHeaders, useCultureSeparator ? ListSeparator : detectSeparator(path),
                                   '\"', '\"', '\0', ValueTrimmingOptions.UnquotedOnly);
            List<string[]> arrayList = arrayListFromCSV(cr);

            return arrayList;
        }

        /// <summary>
        ///     Converts CSV data from a string into a list of string arrays.
        /// </summary>
        /// <param name="text">The CSV data.</param>
        /// <param name="hasHeaders">
        ///     if set to <c>true</c>, the file is assumed to have headers in the first row.
        /// </param>
        /// <param name="useCultureSeparator">
        ///     If <c>true</c>, the method uses the current culture's list separator.
        ///     If <c>false</c>, it tries to automatically detect it.
        /// </param>
        /// <returns>A list of string arrays containing the CSV data.</returns>
        public static List<string[]> ArrayListFromCSVString(string text, bool hasHeaders = true, bool useCultureSeparator = false)
        {
            var cr = new CsvReader(new StringReader(text), hasHeaders,
                                   useCultureSeparator ? ListSeparator : DetectSeparator(new StringReader(text), 1, new[] {',', ';'}),
                                   '\"', '\"', '\0', ValueTrimmingOptions.UnquotedOnly);
            List<string[]> arrayList = arrayListFromCSV(cr);

            return arrayList;
        }

        private static List<Dictionary<string, string>> dictionaryListFromCSV(CsvReader cr)
        {
            List<Dictionary<string, string>> dictList;
            using (cr)
            {
                dictList = new List<Dictionary<string, string>>();

                int fieldCount = cr.FieldCount;
                string[] headers = cr.GetFieldHeaders();

                for (int i = 0; i < headers.Length; i++)
                {
                    var sortingChars = new[] {'^', Convert.ToChar(65533)};
                    char firstChar = Convert.ToChar(headers[i].Substring(0, 1));
                    if (sortingChars.Contains(firstChar))
                    {
                        headers[i] = headers[i].Split(new[] {" ", "\r\n", "\n"}, 2, StringSplitOptions.None)[1];
                        break;
                    }
                }

                int j = 0;
                while (cr.ReadNextRecord())
{'fraction_non_alphanumeric': 0.120559198674065, 'fraction_numerical': 0.004035454348922678, 'mean_word_length': 1.9825918762088974, 'pattern_counts': {'":': 0, '<': 159, '<?xml version=': 0, '>': 157, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7628100', 'n_tokens_mistral': 3628, 'n_tokens_neox': 3529, 'n_words': 1152}