<commit_before>from __future__ import print_function

import glob
import os
import re
import threading

from functools import partial

import pytest

import numpy as np
from numpy.random import rand

from pandas import (DataFrame, MultiIndex, read_csv, Timestamp, Index,
                    date_range, Series)
from pandas.compat import (map, zip, StringIO, BytesIO,
                           is_platform_windows, PY3, reload)
from pandas.io.common import URLError, file_path_to_url
import pandas.io.html
from pandas.io.html import read_html
from pandas._libs.parsers import ParserError

import pandas.util.testing as tm
import pandas.util._test_decorators as td
from pandas.util.testing import makeCustomDataframe as mkdf, network


DATA_PATH = tm.get_data_path()


def assert_framelist_equal(list1, list2, *args, **kwargs):
    assert len(list1) == len(list2), ('lists are not of equal size '
                                      'len(list1) == {0}, '
                                      'len(list2) == {1}'.format(len(list1),
                                                                 len(list2)))
    msg = 'not all list elements are DataFrames'
    both_frames = all(map(lambda x, y: isinstance(x, DataFrame) and
                          isinstance(y, DataFrame), list1, list2))
    assert both_frames, msg
    for frame_i, frame_j in zip(list1, list2):
        tm.assert_frame_equal(frame_i, frame_j, *args, **kwargs)
        assert not frame_i.empty, 'frames are both empty'


@td.skip_if_no('bs4')
def test_bs4_version_fails(monkeypatch):
    import bs4
    monkeypatch.setattr(bs4, '__version__', '4.2')
    with tm.assert_raises_regex(ValueError, "minimum version"):
        read_html(os.path.join(DATA_PATH, "spam.html"), flavor='bs4')


def test_invalid_flavor():
    url = 'google.com'
    with pytest.raises(ValueError):
        read_html(url, 'google', flavor='not a* valid**++ flaver')


@td.skip_if_no('bs4')
@td.skip_if_no('lxml')
def test_same_ordering():
    filename = os.path.join(DATA_PATH, 'valid_markup.html')
    dfs_lxml = read_html(filename, index_col=0, flavor=['lxml'])
    dfs_bs4 = read_html(filename, index_col=0, flavor=['bs4'])
    assert_framelist_equal(dfs_lxml, dfs_bs4)


@pytest.mark.parametrize("flavor", [
    pytest.param('bs4', marks=pytest.mark.skipif(
        not td.safe_import('lxml'), reason='No bs4')),
    pytest.param('lxml', marks=pytest.mark.skipif(
        not td.safe_import('lxml'), reason='No lxml'))], scope="class")
class TestReadHtml(object):
    spam_data = os.path.join(DATA_PATH, 'spam.html')
    spam_data_kwargs = {}
    if PY3:
        spam_data_kwargs['encoding'] = 'UTF-8'
    banklist_data = os.path.join(DATA_PATH, 'banklist.html')

    @pytest.fixture(autouse=True, scope="function")
    def set_defaults(self, flavor, request):
        self.read_html = partial(read_html, flavor=flavor)
        yield

    def test_to_html_compat(self):
        df = mkdf(4, 3, data_gen_f=lambda *args: rand(), c_idx_names=False,
                  r_idx_names=False).applymap('{0:.3f}'.format).astype(float)
        out = df.to_html()
        res = self.read_html(out, attrs={'class': 'dataframe'}, index_col=0)[0]
        tm.assert_frame_equal(res, df)

    @network
    def test_banklist_url(self):
        url = 'http://www.fdic.gov/bank/individual/failed/banklist.html'
        df1 = self.read_html(url, 'First Federal Bank of Florida',
                             attrs={"id": 'table'})
        df2 = self.read_html(url, 'Metcalf Bank', attrs={'id': 'table'})

        assert_framelist_equal(df1, df2)

    @pytest.mark.xfail(reason="no tables found: gh-20685")
    @network
    def test_spam_url(self):
        url = ('http://ndb.nal.usda.gov/ndb/foods/show/1732?fg=&man=&'
               'lfacet=&format=&count=&max=25&offset=&sort=&qlookup=spam')
        df1 = self.read_html(url, '.*Water.*')
        df2 = self.read_html(url, 'Unit')

        assert_framelist_equal(df1, df2)

    @pytest.mark.slow<commit_msg>TST: Updated URL for spam test (#20689)<commit_after>from __future__ import print_function

import glob
import os
import re
import threading

from functools import partial

import pytest

import numpy as np
from numpy.random import rand

from pandas import (DataFrame, MultiIndex, read_csv, Timestamp, Index,
                    date_range, Series)
from pandas.compat import (map, zip, StringIO, BytesIO,
                           is_platform_windows, PY3, reload)
from pandas.io.common import URLError, file_path_to_url
import pandas.io.html
from pandas.io.html import read_html
from pandas._libs.parsers import ParserError

import pandas.util.testing as tm
import pandas.util._test_decorators as td
from pandas.util.testing import makeCustomDataframe as mkdf, network


DATA_PATH = tm.get_data_path()


def assert_framelist_equal(list1, list2, *args, **kwargs):
    assert len(list1) == len(list2), ('lists are not of equal size '
                                      'len(list1) == {0}, '
                                      'len(list2) == {1}'.format(len(list1),
                                                                 len(list2)))
    msg = 'not all list elements are DataFrames'
    both_frames = all(map(lambda x, y: isinstance(x, DataFrame) and
                          isinstance(y, DataFrame), list1, list2))
    assert both_frames, msg
    for frame_i, frame_j in zip(list1, list2):
        tm.assert_frame_equal(frame_i, frame_j, *args, **kwargs)
        assert not frame_i.empty, 'frames are both empty'


@td.skip_if_no('bs4')
def test_bs4_version_fails(monkeypatch):
    import bs4
    monkeypatch.setattr(bs4, '__version__', '4.2')
    with tm.assert_raises_regex(ValueError, "minimum version"):
        read_html(os.path.join(DATA_PATH, "spam.html"), flavor='bs4')


def test_invalid_flavor():
    url = 'google.com'
    with pytest.raises(ValueError):
        read_html(url, 'google', flavor='not a* valid**++ flaver')


@td.skip_if_no('bs4')
@td.skip_if_no('lxml')
def test_same_ordering():
    filename = os.path.join(DATA_PATH, 'valid_markup.html')
    dfs_lxml = read_html(filename, index_col=0, flavor=['lxml'])
    dfs_bs4 = read_html(filename, index_col=0, flavor=['bs4'])
    assert_framelist_equal(dfs_lxml, dfs_bs4)


@pytest.mark.parametrize("flavor", [
    pytest.param('bs4', marks=pytest.mark.skipif(
        not td.safe_import('lxml'), reason='No bs4')),
    pytest.param('lxml', marks=pytest.mark.skipif(
        not td.safe_import('lxml'), reason='No lxml'))], scope="class")
class TestReadHtml(object):
    spam_data = os.path.join(DATA_PATH, 'spam.html')
    spam_data_kwargs = {}
    if PY3:
        spam_data_kwargs['encoding'] = 'UTF-8'
    banklist_data = os.path.join(DATA_PATH, 'banklist.html')

    @pytest.fixture(autouse=True, scope="function")
    def set_defaults(self, flavor, request):
        self.read_html = partial(read_html, flavor=flavor)
        yield

    def test_to_html_compat(self):
        df = mkdf(4, 3, data_gen_f=lambda *args: rand(), c_idx_names=False,
                  r_idx_names=False).applymap('{0:.3f}'.format).astype(float)
        out = df.to_html()
        res = self.read_html(out, attrs={'class': 'dataframe'}, index_col=0)[0]
        tm.assert_frame_equal(res, df)

    @network
    def test_banklist_url(self):
        url = 'http://www.fdic.gov/bank/individual/failed/banklist.html'
        df1 = self.read_html(url, 'First Federal Bank of Florida',
                             attrs={"id": 'table'})
        df2 = self.read_html(url, 'Metcalf Bank', attrs={'id': 'table'})

        assert_framelist_equal(df1, df2)

    @network
    def test_spam_url(self):
        url = ('http://ndb.nal.usda.gov/ndb/foods/show/300772?fg=&man=&'
               'lfacet=&format=&count=&max=25&offset=&sort=&qlookup=spam')
        df1 = self.read_html(url, '.*Water.*')
        df2 = self.read_html(url, 'Unit')

        assert_framelist_equal(df1, df2)

    @pytest.mark.slow
{'fraction_non_alphanumeric': 0.13140339836375078, 'fraction_numerical': 0.01460037759597231, 'mean_word_length': 3.074871794871795, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 16}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1180964', 'n_tokens_mistral': 2807, 'n_tokens_neox': 2715, 'n_words': 553}