<commit_before>import unittest
import warnings
import nose
from nose.tools import assert_equal
from datetime import datetime

import numpy as np
import pandas as pd
from pandas import DataFrame
from pandas.io import data as web
from pandas.io.data import DataReader, SymbolWarning
from pandas.util.testing import (assert_series_equal, assert_produces_warning,
                                 assert_frame_equal, network)
from numpy.testing import assert_array_equal


def assert_n_failed_equals_n_null_columns(wngs, obj, cls=SymbolWarning):
    all_nan_cols = pd.Series(dict((k, pd.isnull(v).all()) for k, v in
                                  obj.iteritems()))
    n_all_nan_cols = all_nan_cols.sum()
    valid_warnings = pd.Series([wng for wng in wngs if isinstance(wng, cls)])
    assert_equal(len(valid_warnings), n_all_nan_cols)
    failed_symbols = all_nan_cols[all_nan_cols].index
    msgs = valid_warnings.map(lambda x: x.message)
    assert msgs.str.contains('|'.join(failed_symbols)).all()


class TestGoogle(unittest.TestCase):

    @network
    def test_google(self):
        # asserts that google is minimally working and that it throws
        # an exception when DataReader can't get a 200 response from
        # google
        start = datetime(2010, 1, 1)
        end = datetime(2013, 01, 27)

        self.assertEquals(
            web.DataReader("F", 'google', start, end)['Close'][-1],
            13.68)

        self.assertRaises(Exception, web.DataReader, "NON EXISTENT TICKER",
                          'google', start, end)

    @network
    def test_get_quote_fails(self):
        self.assertRaises(NotImplementedError, web.get_quote_google,
                          pd.Series(['GOOG', 'AAPL', 'GOOG']))

    @network
    def test_get_goog_volume(self):
        df = web.get_data_google('GOOG')
        self.assertEqual(df.Volume.ix['OCT-08-2010'], 2863473)

    @network
    def test_get_multi1(self):
        sl = ['AAPL', 'AMZN', 'GOOG']
        pan = web.get_data_google(sl, '2012')

        def testit():
            ts = pan.Close.GOOG.index[pan.Close.AAPL > pan.Close.GOOG]
            self.assertEquals(ts[0].dayofyear, 96)

        if (hasattr(pan, 'Close') and hasattr(pan.Close, 'GOOG') and
            hasattr(pan.Close, 'AAPL')):
            testit()
        else:
            self.assertRaises(AttributeError, testit)

    @network
    def test_get_multi2(self):
        with warnings.catch_warnings(record=True) as w:
            pan = web.get_data_google(['GE', 'MSFT', 'INTC'], 'JAN-01-12',
                                      'JAN-31-12')
            result = pan.Close.ix['01-18-12']
            assert_n_failed_equals_n_null_columns(w, result)

            # sanity checking

            assert np.issubdtype(result.dtype, np.floating)
            result = pan.Open.ix['Jan-15-12':'Jan-20-12']
            self.assertEqual((4, 3), result.shape)
            assert_n_failed_equals_n_null_columns(w, result)


class TestYahoo(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        try:
            import lxml
        except ImportError:
            raise nose.SkipTest

    @network
    def test_yahoo(self):
        # asserts that yahoo is minimally working and that it throws
        # an exception when DataReader can't get a 200 response from
        # yahoo
        start = datetime(2010, 1, 1)
        end = datetime(2013, 01, 27)

        self.assertEquals( web.DataReader("F", 'yahoo', start,
                                          end)['Close'][-1], 13.68)

    @network
    def test_yahoo_fails(self):
        start = datetime(2010, 1, 1)
        end = datetime(2013, 01, 27)
        self.assertRaises(Exception, web.DataReader, "NON EXISTENT TICKER",
                          'yahoo', start, end)

    @network
    def test_get_quote_series(self):
        df = web.get_quote_yahoo(pd.Series(['GOOG', 'AAPL', 'GOOG']))
        assert_series_equal(df.ix[0], df.ix[2])

    @network
    def test_get_quote_string(self):
        df = web.get_quote_yahoo('GOOG')
        df2 = web.get_quote_yahoo('GOOG')
        assert_frame_equal(df, df2)

    @network
    def test_get_quote_stringlist(self):
        df = web.get_quote_yahoo(['GOOG', 'AAPL', 'GOOG'])
        assert_series_equal(df.ix[0], df.ix[2])

    @network
    def test_get_components_dow_jones(self):
        df = web.get_components_yahoo('^DJI') #Dow Jones
        assert isinstance(df, pd.DataFrame)
        self.assertEqual(len(df), 30)

    @network
    def test_get_components_dax(self):
        df = web.get_components_yahoo('^GDAXI') #DAX
        assert isinstance(df, pd.DataFrame)
        self.assertEqual(len(df), 30)
        self.assertEqual(df[df.name.str.contains('adidas', case=False)].index,
                         'ADS.DE')

    @network
    def test_get_components_nasdaq_100(self):
        df = web.get_components_yahoo('^NDX') #NASDAQ-100
        assert isinstance(df, pd.DataFrame)
        # Usual culprits, should be around for a while
        assert 'AAPL' in df.index
        assert 'GOOG' in df.index<commit_msg>TST: remove double call to yahoo finance<commit_after>import unittest
import warnings
import nose
from nose.tools import assert_equal
from datetime import datetime

import numpy as np
import pandas as pd
from pandas import DataFrame
from pandas.io import data as web
from pandas.io.data import DataReader, SymbolWarning
from pandas.util.testing import (assert_series_equal, assert_produces_warning,
                                 assert_frame_equal, network)
from numpy.testing import assert_array_equal


def assert_n_failed_equals_n_null_columns(wngs, obj, cls=SymbolWarning):
    all_nan_cols = pd.Series(dict((k, pd.isnull(v).all()) for k, v in
                                  obj.iteritems()))
    n_all_nan_cols = all_nan_cols.sum()
    valid_warnings = pd.Series([wng for wng in wngs if isinstance(wng, cls)])
    assert_equal(len(valid_warnings), n_all_nan_cols)
    failed_symbols = all_nan_cols[all_nan_cols].index
    msgs = valid_warnings.map(lambda x: x.message)
    assert msgs.str.contains('|'.join(failed_symbols)).all()


class TestGoogle(unittest.TestCase):

    @network
    def test_google(self):
        # asserts that google is minimally working and that it throws
        # an exception when DataReader can't get a 200 response from
        # google
        start = datetime(2010, 1, 1)
        end = datetime(2013, 01, 27)

        self.assertEquals(
            web.DataReader("F", 'google', start, end)['Close'][-1],
            13.68)

        self.assertRaises(Exception, web.DataReader, "NON EXISTENT TICKER",
                          'google', start, end)

    @network
    def test_get_quote_fails(self):
        self.assertRaises(NotImplementedError, web.get_quote_google,
                          pd.Series(['GOOG', 'AAPL', 'GOOG']))

    @network
    def test_get_goog_volume(self):
        df = web.get_data_google('GOOG')
        self.assertEqual(df.Volume.ix['OCT-08-2010'], 2863473)

    @network
    def test_get_multi1(self):
        sl = ['AAPL', 'AMZN', 'GOOG']
        pan = web.get_data_google(sl, '2012')

        def testit():
            ts = pan.Close.GOOG.index[pan.Close.AAPL > pan.Close.GOOG]
            self.assertEquals(ts[0].dayofyear, 96)

        if (hasattr(pan, 'Close') and hasattr(pan.Close, 'GOOG') and
            hasattr(pan.Close, 'AAPL')):
            testit()
        else:
            self.assertRaises(AttributeError, testit)

    @network
    def test_get_multi2(self):
        with warnings.catch_warnings(record=True) as w:
            pan = web.get_data_google(['GE', 'MSFT', 'INTC'], 'JAN-01-12',
                                      'JAN-31-12')
            result = pan.Close.ix['01-18-12']
            assert_n_failed_equals_n_null_columns(w, result)

            # sanity checking

            assert np.issubdtype(result.dtype, np.floating)
            result = pan.Open.ix['Jan-15-12':'Jan-20-12']
            self.assertEqual((4, 3), result.shape)
            assert_n_failed_equals_n_null_columns(w, result)


class TestYahoo(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        try:
            import lxml
        except ImportError:
            raise nose.SkipTest

    @network
    def test_yahoo(self):
        # asserts that yahoo is minimally working and that it throws
        # an exception when DataReader can't get a 200 response from
        # yahoo
        start = datetime(2010, 1, 1)
        end = datetime(2013, 01, 27)

        self.assertEquals( web.DataReader("F", 'yahoo', start,
                                          end)['Close'][-1], 13.68)

    @network
    def test_yahoo_fails(self):
        start = datetime(2010, 1, 1)
        end = datetime(2013, 01, 27)
        self.assertRaises(Exception, web.DataReader, "NON EXISTENT TICKER",
                          'yahoo', start, end)

    @network
    def test_get_quote_series(self):
        df = web.get_quote_yahoo(pd.Series(['GOOG', 'AAPL', 'GOOG']))
        assert_series_equal(df.ix[0], df.ix[2])

    @network
    def test_get_quote_string(self):
        df = web.get_quote_yahoo('GOOG')

    @network
    def test_get_quote_stringlist(self):
        df = web.get_quote_yahoo(['GOOG', 'AAPL', 'GOOG'])
        assert_series_equal(df.ix[0], df.ix[2])

    @network
    def test_get_components_dow_jones(self):
        df = web.get_components_yahoo('^DJI') #Dow Jones
        assert isinstance(df, pd.DataFrame)
        self.assertEqual(len(df), 30)

    @network
    def test_get_components_dax(self):
        df = web.get_components_yahoo('^GDAXI') #DAX
        assert isinstance(df, pd.DataFrame)
        self.assertEqual(len(df), 30)
        self.assertEqual(df[df.name.str.contains('adidas', case=False)].index,
                         'ADS.DE')

    @network
    def test_get_components_nasdaq_100(self):
        df = web.get_components_yahoo('^NDX') #NASDAQ-100
        assert isinstance(df, pd.DataFrame)
        # Usual culprits, should be around for a while
        assert 'AAPL' in df.index
        assert 'GOOG' in df.index
{'fraction_non_alphanumeric': 0.11268442420041588, 'fraction_numerical': 0.02356668977126448, 'mean_word_length': 2.4388832141641132, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1180963', 'n_tokens_mistral': 3428, 'n_tokens_neox': 3181, 'n_words': 701}