<commit_before>import asyncio
import os
import io
import json
import unittest

import aiohttp
import asynctest
from aioresponses import aioresponses

from buildhub import inventory_to_records


here = os.path.dirname(__file__)


class LongResponse:
    def __init__(*args, **kwargs):
        pass
    async def __call__(*args, **kwargs):
        return self
    async def __aenter__(self, *args):
        return self
    async def __aexit__(self, *args):
        pass
    async def json(self):
        return await asyncio.sleep(10000)


class FetchJsonTest(asynctest.TestCase):
    url = 'http://test.example.com'
    data = {'foo': 'bar'}

    async def setUp(self):
        self.session = aiohttp.ClientSession(loop=self.loop)
        self.addCleanup(self.session.close)

        mocked = aioresponses()
        mocked.start()
        mocked.get(self.url, payload=self.data)
        self.addCleanup(mocked.stop)

    async def test_returns_json_response(self):
        received = await inventory_to_records.fetch_json(self.session, self.url)
        assert received == self.data

    async def test_raises_timeout_response(self):
        with asynctest.patch.object(self.session, "get", LongResponse):
            with self.assertRaises(asyncio.TimeoutError):
                await inventory_to_records.fetch_json(self.session, self.url, 0.1)


class CsvToRecordsTest(asynctest.TestCase):
    def test_load_simple_file(self):
        filename = os.path.join(here, "data", "inventory-simple.csv")
        stdout = io.StringIO()

        future = inventory_to_records.csv_to_records(self.loop, filename, stdout)
        self.loop.run_until_complete(future)

        output = stdout.getvalue()
        records = [json.loads(o) for o in output.split("\n") if o]
        assert records == [{
            "id": "firefox_nightly_2017-05-15-10-02-38_55-0a1_linux-x86_64_en-us",
            "build": {
                "id": "20170515100238",
                "date": "2017-05-15T10:02:38Z"
            },
            "source": {<commit_msg>Add test for fetch_listing()<commit_after>import asyncio
import os
import io
import json
import unittest

import aiohttp
import asynctest
from aioresponses import aioresponses

from buildhub import inventory_to_records


here = os.path.dirname(__file__)


class LongResponse:
    def __init__(*args, **kwargs):
        pass
    async def __call__(*args, **kwargs):
        return self
    async def __aenter__(self, *args):
        return self
    async def __aexit__(self, *args):
        pass
    async def json(self):
        return await asyncio.sleep(10000)


class FetchJsonTest(asynctest.TestCase):
    url = 'http://test.example.com'
    data = {'foo': 'bar'}

    async def setUp(self):
        self.session = aiohttp.ClientSession(loop=self.loop)
        self.addCleanup(self.session.close)

        mocked = aioresponses()
        mocked.start()
        mocked.get(self.url, payload=self.data)
        self.addCleanup(mocked.stop)

    async def test_returns_json_response(self):
        received = await inventory_to_records.fetch_json(self.session, self.url)
        assert received == self.data

    async def test_raises_timeout_response(self):
        with asynctest.patch.object(self.session, "get", LongResponse):
            with self.assertRaises(asyncio.TimeoutError):
                await inventory_to_records.fetch_json(self.session, self.url, 0.1)


class FetchListingTest(asynctest.TestCase):
    url = 'http://test.example.com'

    async def setUp(self):
        self.session = aiohttp.ClientSession(loop=self.loop)
        self.addCleanup(self.session.close)

    async def test_returns_tuple_with_directories_and_files(self):
        with aioresponses() as m:
            m.get(self.url, payload={
                "prefixes": ["a/", "b/"],
                "files": [{"name": "foo.txt"}]
            })
            received = await inventory_to_records.fetch_listing(self.session, self.url)
            assert received == (["a/", "b/"], [{"name": "foo.txt"}])

    async def test_raises_valueerror_if_bad_json(self):
        with aioresponses() as m:
            m.get(self.url, payload={
                "prfixes": ["a/", "b/"],
            })
            with self.assertRaises(ValueError):
                await inventory_to_records.fetch_listing(self.session, self.url)

    async def test_raises_valueerror_if_html(self):
        with aioresponses() as m:
            m.get(self.url, body="<html></html>")
            with self.assertRaises(ValueError):
                await inventory_to_records.fetch_listing(self.session, self.url)


class CsvToRecordsTest(asynctest.TestCase):
    def test_load_simple_file(self):
        filename = os.path.join(here, "data", "inventory-simple.csv")
        stdout = io.StringIO()

        future = inventory_to_records.csv_to_records(self.loop, filename, stdout)
        self.loop.run_until_complete(future)

        output = stdout.getvalue()
        records = [json.loads(o) for o in output.split("\n") if o]
        assert records == [{
            "id": "firefox_nightly_2017-05-15-10-02-38_55-0a1_linux-x86_64_en-us",
            "build": {
                "id": "20170515100238",
                "date": "2017-05-15T10:02:38Z"
            },
            "source": {
{'fraction_non_alphanumeric': 0.11635398900890657, 'fraction_numerical': 0.02160318362706083, 'mean_word_length': 2.6525951557093426, 'pattern_counts': {'":': 15, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2874898', 'n_tokens_mistral': 1752, 'n_tokens_neox': 1650, 'n_words': 329}