<commit_before># encoding: utf-8

"""
.. codeauthor:: <NAME> <<EMAIL>>
"""

from __future__ import print_function, unicode_literals

import os.path
import platform
from textwrap import dedent

import pytablereader as ptr
import pytest
import responses
import six
from pytablereader.interface import TableLoader
from pytablewriter import dump_tabledata
from tabledata import TableData


class Test_TableUrlLoader_get_format_name_list(object):
    def test_normal(self):
        format_name_list = ptr.TableUrlLoader.get_format_name_list()

        assert format_name_list == [
            "csv",
            "excel",
            "html",
            "json",
            "json_lines",
            "jsonl",
            "ldjson",
            "ltsv",
            "markdown",
            "mediawiki",
            "ndjson",
            "sqlite",
            "ssv",
            "tsv",
        ]


class Test_TableUrlLoader_constructor(object):
    @responses.activate
    @pytest.mark.parametrize(
        ["value", "format_name", "expected"],
        [
            [
                "https://raw.githubusercontent.com/valid/test/data/validext.csv",
                None,
                ptr.CsvTableTextLoader,
            ],
            [
                "https://raw.githubusercontent.com/valid/test/data/validext.csv/",
                None,
                ptr.CsvTableTextLoader,
            ],
            [
                "https://raw.githubusercontent.com/valid/test/data/validext.xlsx",
                None,
                ptr.ExcelTableFileLoader,
            ],
            [
                "https://raw.githubusercontent.com/valid/test/data/validext.html",
                None,
                ptr.HtmlTableTextLoader,
            ],
            [
                "https://raw.githubusercontent.com/valid/test/data/validext.htm/",
                None,
                ptr.HtmlTableTextLoader,
            ],
            [
                "https://raw.githubusercontent.com/valid/test/data/validext.asp",
                None,
                ptr.HtmlTableTextLoader,
            ],
            [
                "https://raw.githubusercontent.com/valid/test/data/validext.aspx/",
                None,
                ptr.HtmlTableTextLoader,
            ],
            [
                "https://raw.githubusercontent.com/valid/test/data/validext.json",
                None,
                ptr.JsonTableTextLoader,
            ],
            [
                "https://raw.githubusercontent.com/valid/test/data/validext.jsonl",
                None,
                ptr.JsonLinesTableTextLoader,
            ],
            [
                "https://raw.githubusercontent.com/valid/test/data/validext.ldjson",
                None,
                ptr.JsonLinesTableTextLoader,
            ],
            ["https://github.com/validext.ltsv", None, ptr.LtsvTableTextLoader],
            ["https://github.com/validext.md", None, ptr.MarkdownTableTextLoader],
            [
                "https://raw.githubusercontent.com/valid/test/data/validext.ndjson",
                None,
                ptr.JsonLinesTableTextLoader,
            ],
            [
                "https://raw.githubusercontent.com/valid/test/data/validext.sqlite",
                None,
                ptr.SqliteFileLoader,
            ],
            [
                "https://raw.githubusercontent.com/valid/test/data/validext.sqlite3",
                None,
                ptr.SqliteFileLoader,
            ],
            [
                "https://raw.githubusercontent.com/valid/test/data/validext.tsv",
                None,
                ptr.TsvTableTextLoader,
            ],
            ["https://github.com/validext.txt", "csv", ptr.CsvTableTextLoader],
            ["https://github.com/validext.txt", "html", ptr.HtmlTableTextLoader],
            ["https://github.com/validext.txt", "json", ptr.JsonTableTextLoader],
            ["https://github.com/validext.txt", "jsonl", ptr.JsonLinesTableTextLoader],
            ["https://github.com/validext.txt", "json_lines", ptr.JsonLinesTableTextLoader],
            ["https://github.com/validext.txt", "ldjson", ptr.JsonLinesTableTextLoader],
            ["https://github.com/invalidext.txt", "markdown", ptr.MarkdownTableTextLoader],
            ["https://github.com/invalidext.txt", "mediawiki", ptr.MediaWikiTableTextLoader],
            ["https://github.com/validext.txt", "ndjson", ptr.JsonLinesTableTextLoader],
            ["https://github.com/validext.txt", "tsv", ptr.TsvTableTextLoader],
        ],
    )
    def test_normal(self, value, format_name, expected):
        if six.PY2 and platform.system() == "Windows":
            pytest.skip()

        responses.add(
            responses.GET,
            value,
            body="""{}""",
            content_type="text/plain; charset=utf-8",
            status=200,
        )

        loader = ptr.TableUrlLoader(value, format_name)
        expected_loader = expected("")

        assert loader.source_type == expected_loader.source_type
        assert loader.format_name == expected_loader.format_name

    @responses.activate
    @pytest.mark.parametrize(
        ["value", "format_name", "expected"],
        [
            [None, None, ValueError],
            ["", None, ptr.UrlError],
            ["https://github.com/", None, ptr.UrlError],
            ["/tmp/test.txt", None, ptr.UrlError],
            ["c:\\tmp\test.txt", None, ptr.UrlError],
            [
                "https://raw.githubusercontent.com/invalid/test/data/invalidext.txt",
                None,
                ptr.LoaderNotFoundError,
            ],
            [
                "https://raw.githubusercontent.com/invalid/test/data/notexist.json",
                None,
                ptr.HTTPError,
            ],
            [
                "https://raw.githubusercontent.com/invalid/test/data/invalidext.txt",
                "invalidformat",
                ptr.LoaderNotFoundError,
            ],
        ],
    )
    def test_exception(self, value, format_name, expected):
        responses.add(responses.GET, value, body="""404: Not Found""", status=404)

        with pytest.raises(expected):
            ptr.TableUrlLoader(value, format_name)


class Test_TableUrlLoader_load(object):
    def setup_method(self, method):
        TableLoader.clear_table_count()

    @responses.activate
    @pytest.mark.parametrize(
        ["url", "format_name"],
        [
            ["https://raw.githubusercontent.com/valid/test/data/validdata.csv", None],
            ["https://raw.githubusercontent.com/valid/test/data/validdata.txt", "csv"],<commit_msg>Fix a test case<commit_after># encoding: utf-8

"""
.. codeauthor:: <NAME> <<EMAIL>>
"""

from __future__ import print_function, unicode_literals

import os.path
import platform
from textwrap import dedent

import pytablereader as ptr
import pytest
import responses
import six
from pytablereader.interface import TableLoader
from pytablewriter import dump_tabledata
from tabledata import TableData


class Test_TableUrlLoader_get_format_name_list(object):
    def test_normal(self):
        format_name_list = ptr.TableUrlLoader.get_format_name_list()

        assert format_name_list == [
            "csv",
            "excel",
            "html",
            "json",
            "json_lines",
            "jsonl",
            "ldjson",
            "ltsv",
            "markdown",
            "mediawiki",
            "ndjson",
            "sqlite",
            "ssv",
            "tsv",
        ]


class Test_TableUrlLoader_constructor(object):
    @responses.activate
    @pytest.mark.parametrize(
        ["value", "format_name", "expected"],
        [
            [
                "https://raw.githubusercontent.com/valid/test/data/validext.csv",
                None,
                ptr.CsvTableTextLoader,
            ],
            [
                "https://raw.githubusercontent.com/valid/test/data/validext.csv/",
                None,
                ptr.CsvTableTextLoader,
            ],
            [
                "https://raw.githubusercontent.com/valid/test/data/validext.xlsx",
                None,
                ptr.ExcelTableFileLoader,
            ],
            [
                "https://raw.githubusercontent.com/valid/test/data/validext.html",
                None,
                ptr.HtmlTableTextLoader,
            ],
            [
                "https://raw.githubusercontent.com/valid/test/data/validext.htm/",
                None,
                ptr.HtmlTableTextLoader,
            ],
            [
                "https://raw.githubusercontent.com/valid/test/data/validext.asp",
                None,
                ptr.HtmlTableTextLoader,
            ],
            [
                "https://raw.githubusercontent.com/valid/test/data/validext.aspx/",
                None,
                ptr.HtmlTableTextLoader,
            ],
            [
                "https://raw.githubusercontent.com/valid/test/data/validext.json",
                None,
                ptr.JsonTableTextLoader,
            ],
            [
                "https://raw.githubusercontent.com/valid/test/data/validext.jsonl",
                None,
                ptr.JsonLinesTableTextLoader,
            ],
            [
                "https://raw.githubusercontent.com/valid/test/data/validext.ldjson",
                None,
                ptr.JsonLinesTableTextLoader,
            ],
            ["https://github.com/validext.ltsv", None, ptr.LtsvTableTextLoader],
            ["https://github.com/validext.md", None, ptr.MarkdownTableTextLoader],
            [
                "https://raw.githubusercontent.com/valid/test/data/validext.ndjson",
                None,
                ptr.JsonLinesTableTextLoader,
            ],
            [
                "https://raw.githubusercontent.com/valid/test/data/validext.sqlite",
                None,
                ptr.SqliteFileLoader,
            ],
            [
                "https://raw.githubusercontent.com/valid/test/data/validext.sqlite3",
                None,
                ptr.SqliteFileLoader,
            ],
            [
                "https://raw.githubusercontent.com/valid/test/data/validext.tsv",
                None,
                ptr.TsvTableTextLoader,
            ],
            ["https://github.com/validext.txt", "csv", ptr.CsvTableTextLoader],
            ["https://github.com/validext.txt", "html", ptr.HtmlTableTextLoader],
            ["https://github.com/validext.txt", "json", ptr.JsonTableTextLoader],
            ["https://github.com/validext.txt", "jsonl", ptr.JsonLinesTableTextLoader],
            ["https://github.com/validext.txt", "json_lines", ptr.JsonLinesTableTextLoader],
            ["https://github.com/validext.txt", "ldjson", ptr.JsonLinesTableTextLoader],
            ["https://github.com/invalidext.txt", "markdown", ptr.MarkdownTableTextLoader],
            ["https://github.com/invalidext.txt", "mediawiki", ptr.MediaWikiTableTextLoader],
            ["https://github.com/validext.txt", "ndjson", ptr.JsonLinesTableTextLoader],
            ["https://github.com/validext.txt", "tsv", ptr.TsvTableTextLoader],
        ],
    )
    def test_normal(self, value, format_name, expected):
        responses.add(
            responses.GET,
            value,
            body="""{}""",
            content_type="text/plain; charset=utf-8",
            status=200,
        )

        loader = ptr.TableUrlLoader(value, format_name)
        expected_loader = expected("")

        assert loader.source_type == expected_loader.source_type
        assert loader.format_name == expected_loader.format_name

    @responses.activate
    @pytest.mark.parametrize(
        ["value", "format_name", "expected"],
        [
            [None, None, ValueError],
            ["", None, ptr.UrlError],
            ["https://github.com/", None, ptr.UrlError],
            ["/tmp/test.txt", None, ptr.UrlError],
            ["c:\\tmp\test.txt", None, ptr.UrlError],
            [
                "https://raw.githubusercontent.com/invalid/test/data/invalidext.txt",
                None,
                ptr.LoaderNotFoundError,
            ],
            [
                "https://raw.githubusercontent.com/invalid/test/data/notexist.json",
                None,
                ptr.HTTPError,
            ],
            [
                "https://raw.githubusercontent.com/invalid/test/data/invalidext.txt",
                "invalidformat",
                ptr.LoaderNotFoundError,
            ],
        ],
    )
    def test_exception(self, value, format_name, expected):
        if six.PY2 and platform.system() == "Windows":
            pytest.skip()

        responses.add(responses.GET, value, body="""404: Not Found""", status=404)

        with pytest.raises(expected):
            ptr.TableUrlLoader(value, format_name)


class Test_TableUrlLoader_load(object):
    def setup_method(self, method):
        TableLoader.clear_table_count()

    @responses.activate
    @pytest.mark.parametrize(
        ["url", "format_name"],
        [
            ["https://raw.githubusercontent.com/valid/test/data/validdata.csv", None],
            ["https://raw.githubusercontent.com/valid/test/data/validdata.txt", "csv"],
{'fraction_non_alphanumeric': 0.12701279157261097, 'fraction_numerical': 0.0019563581640331075, 'mean_word_length': 1.6184003152088258, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 64, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1167993', 'n_tokens_mistral': 3850, 'n_tokens_neox': 3727, 'n_words': 470}