<commit_before>﻿Imports NUnit.Framework

Namespace CompuMaster.Test.Data

    <TestFixture(Category:="CSV")> Public Class CsvTest
        Public Sub New()
        End Sub

        Private ReadOnly _OriginCulture As System.Globalization.CultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture
        <TearDown> Public Sub ResetCulture()
            System.Threading.Thread.CurrentThread.CurrentCulture = _OriginCulture
        End Sub

        <Test> Public Sub ReadDataTableFromCsvUrlWithTls12Required()
            Dim Url As String = "https://data.cityofnewyork.us/api/views/kku6-nxdu/rows.csv?accessType=DOWNLOAD"
            Dim CsvCulture As System.Globalization.CultureInfo = System.Globalization.CultureInfo.CreateSpecificCulture("en-US")
            Dim FileEncoding As System.Text.Encoding = Nothing
            Dim dt As DataTable = CompuMaster.Data.Csv.ReadDataTableFromCsvFile(Url, True, FileEncoding, CsvCulture, """"c, False, True)
            Console.WriteLine(CompuMaster.Data.DataTables.ConvertToPlainTextTableFixedColumnWidths(dt))
            Assert.Greater(dt.Columns.Count, 0)
            Assert.Greater(dt.Rows.Count, 0)
        End Sub

        ''' <summary>
        ''' Test from a mini-webserver providing a CSV download with missing response header content-type/charset 
        ''' </summary>
        ''' <remarks>
        ''' The CSV file is returned as UTF-8 bytes
        ''' </remarks>
        <Test> Public Sub ReadDataTableFromCsvUrlAtLocalhostWithContentTypeButWithoutCharset(<Values(Nothing, "text/csv", "text/csv; charset=utf-8")> headerContentType As String)
            Dim Url As String = "http://localhost:8035/"
            Dim CsvCulture As System.Globalization.CultureInfo = System.Globalization.CultureInfo.CreateSpecificCulture("en-US")
            Dim FileEncoding As System.Text.Encoding = Nothing
            Dim Headers As New System.Collections.Specialized.NameValueCollection
            If headerContentType <> Nothing Then
                Headers("content-type") = headerContentType
            End If
            Dim ws As New CompuMaster.Test.Tools.TinyWebServerAdvanced.WebServer(AddressOf ReadDataTableLocalhostTestWebserver, Headers, Url)
            Try
                ws.Run()
                Dim dt As DataTable
                'Test 1
                dt = CompuMaster.Data.Csv.ReadDataTableFromCsvFile(Url, True, FileEncoding, CsvCulture, """"c, False, True)
                Console.WriteLine(CompuMaster.Data.DataTables.ConvertToPlainTextTableFixedColumnWidths(dt))
                Assert.Greater(dt.Columns.Count, 0)
                Assert.Greater(dt.Rows.Count, 0)
                'Test 2
                dt = CompuMaster.Data.Csv.ReadDataTableFromCsvFile(Url, True, "", ","c, """"c, False, True)
                Console.WriteLine(CompuMaster.Data.DataTables.ConvertToPlainTextTableFixedColumnWidths(dt))
                Assert.Greater(dt.Columns.Count, 0)
                Assert.Greater(dt.Rows.Count, 0)
                'Test 3
                dt = CompuMaster.Data.Csv.ReadDataTableFromCsvFile(Url, True, CType(Nothing, String), ","c, """"c, False, True)
                Console.WriteLine(CompuMaster.Data.DataTables.ConvertToPlainTextTableFixedColumnWidths(dt))
                Assert.Greater(dt.Columns.Count, 0)
                Assert.Greater(dt.Rows.Count, 0)
                'Test 4
                dt = CompuMaster.Data.Csv.ReadDataTableFromCsvFile(Url, True, CompuMaster.Data.Csv.ReadLineEncodings.RowBreakCrLfOrCr_CellLineBreakLf, CompuMaster.Data.Csv.ReadLineEncodingAutoConversion.NoAutoConversion, "", ","c, """"c, False, False)
                Console.WriteLine(CompuMaster.Data.DataTables.ConvertToPlainTextTableFixedColumnWidths(dt))
                Assert.Greater(dt.Columns.Count, 0)
                Assert.Greater(dt.Rows.Count, 0)
            Finally
                ws.Stop()
            End Try
        End Sub
<commit_msg>changed test to allow execution of test variants in parallel (requires different TCP listener ports)<commit_after>﻿Imports NUnit.Framework

Namespace CompuMaster.Test.Data

    <TestFixture(Category:="CSV")> Public Class CsvTest
        Public Sub New()
        End Sub

        Private ReadOnly _OriginCulture As System.Globalization.CultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture
        <TearDown> Public Sub ResetCulture()
            System.Threading.Thread.CurrentThread.CurrentCulture = _OriginCulture
        End Sub

        <Test> Public Sub ReadDataTableFromCsvUrlWithTls12Required()
            Dim Url As String = "https://data.cityofnewyork.us/api/views/kku6-nxdu/rows.csv?accessType=DOWNLOAD"
            Dim CsvCulture As System.Globalization.CultureInfo = System.Globalization.CultureInfo.CreateSpecificCulture("en-US")
            Dim FileEncoding As System.Text.Encoding = Nothing
            Dim dt As DataTable = CompuMaster.Data.Csv.ReadDataTableFromCsvFile(Url, True, FileEncoding, CsvCulture, """"c, False, True)
            Console.WriteLine(CompuMaster.Data.DataTables.ConvertToPlainTextTableFixedColumnWidths(dt))
            Assert.Greater(dt.Columns.Count, 0)
            Assert.Greater(dt.Rows.Count, 0)
        End Sub

        ''' <summary>
        ''' Test from a mini-webserver providing a CSV download with missing response header content-type/charset 
        ''' </summary>
        ''' <remarks>
        ''' The CSV file is returned as UTF-8 bytes
        ''' </remarks>
        <Test> Public Sub ReadDataTableFromCsvUrlAtLocalhostWithContentTypeButWithoutCharset(<Values(0, 1, 2)> headerContentTypeVariantToExecute As Integer)
            Dim HeaderContentTypeVariants As String() = New String() {Nothing, "text/csv", "text/csv; charset=utf-8"}
            Dim HeaderContentType As String = HeaderContentTypeVariants(headerContentTypeVariantToExecute)
            Dim Url As String = "http://localhost:" & 8035 + headerContentTypeVariantToExecute & "/"
            Dim CsvCulture As System.Globalization.CultureInfo = System.Globalization.CultureInfo.CreateSpecificCulture("en-US")
            Dim FileEncoding As System.Text.Encoding = Nothing
            Dim Headers As New System.Collections.Specialized.NameValueCollection
            If HeaderContentType <> Nothing Then
                Headers("content-type") = HeaderContentType
            End If
            Dim ws As New CompuMaster.Test.Tools.TinyWebServerAdvanced.WebServer(AddressOf ReadDataTableLocalhostTestWebserver, Headers, Url)
            Try
                ws.Run()
                Dim dt As DataTable
                'Test 1
                dt = CompuMaster.Data.Csv.ReadDataTableFromCsvFile(Url, True, FileEncoding, CsvCulture, """"c, False, True)
                Console.WriteLine(CompuMaster.Data.DataTables.ConvertToPlainTextTableFixedColumnWidths(dt))
                Assert.Greater(dt.Columns.Count, 0)
                Assert.Greater(dt.Rows.Count, 0)
                'Test 2
                dt = CompuMaster.Data.Csv.ReadDataTableFromCsvFile(Url, True, "", ","c, """"c, False, True)
                Console.WriteLine(CompuMaster.Data.DataTables.ConvertToPlainTextTableFixedColumnWidths(dt))
                Assert.Greater(dt.Columns.Count, 0)
                Assert.Greater(dt.Rows.Count, 0)
                'Test 3
                dt = CompuMaster.Data.Csv.ReadDataTableFromCsvFile(Url, True, CType(Nothing, String), ","c, """"c, False, True)
                Console.WriteLine(CompuMaster.Data.DataTables.ConvertToPlainTextTableFixedColumnWidths(dt))
                Assert.Greater(dt.Columns.Count, 0)
                Assert.Greater(dt.Rows.Count, 0)
                'Test 4
                dt = CompuMaster.Data.Csv.ReadDataTableFromCsvFile(Url, True, CompuMaster.Data.Csv.ReadLineEncodings.RowBreakCrLfOrCr_CellLineBreakLf, CompuMaster.Data.Csv.ReadLineEncodingAutoConversion.NoAutoConversion, "", ","c, """"c, False, False)
                Console.WriteLine(CompuMaster.Data.DataTables.ConvertToPlainTextTableFixedColumnWidths(dt))
                Assert.Greater(dt.Columns.Count, 0)
                Assert.Greater(dt.Rows.Count, 0)
            Finally
                ws.Stop()
            End Try
        End Sub

{'fraction_non_alphanumeric': 0.09398680029332682, 'fraction_numerical': 0.005988755805426546, 'mean_word_length': 2.6352732119058198, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7233594', 'n_tokens_mistral': 2346, 'n_tokens_neox': 2227, 'n_words': 446}