<commit_before>#!/usr/bin/env python
from unittest import TestCase, main
from json import loads, dumps
from subprocess import Popen, PIPE
from textwrap import dedent
import sqlite3
import os
import shutil
import datetime
import urllib2
import re

# This library
import scraperwiki


class TestDb(TestCase):
    DBNAME = 'scraperwiki.sqlite'

    def setUp(self):
        self.cleanUp()
        scraperwiki.sqlite._connect(self.DBNAME)

    def tearDown(self):
        self.cleanUp()

    def cleanUp(self):
        "Clean up temporary files, then reinitialize."
        if self.DBNAME != ':memory:':
            try:
                os.remove(self.DBNAME)
            except OSError:
                pass


class TestException(TestDb):

    def testExceptionSaved(self):
        script = dedent("""
            import scraperwiki.runlog
            print scraperwiki.runlog.setup()
            raise ValueError
        """)
        process = Popen(["python", "-c", script],
                        stdout=PIPE, stderr=PIPE, stdin=open("/dev/null"))
        stdout, stderr = process.communicate()

        assert 'Traceback' in stderr, "stderr should contain the original Python traceback"
        match = re.match(r'^\w{8}-\w{4}-\w{4}-\w{4}-\w{12}', stdout)
        assert match, "runlog.setup() should return a run_id"

        l = scraperwiki.sqlite.select(
            "exception_type, run_id, time from _sw_runlog order by time desc limit 1")

        # Check that some record is stored.
        assert l
        # Check that the exception name appears.
        assert 'ValueError' in l[0][
            'exception_type'], "runlog should save exception types to the database"
        # Check that the run_id from earlier has been saved.
        assert match.group() == l[0].get(
            'run_id'), "runlog should save a run_id to the database"
        # Check that the time recorded is relatively recent.
        time_str = l[0]['time']
        then = datetime.datetime.strptime(time_str, '%Y-%m-%d %H:%M:%S.%f')
        assert (datetime.datetime.now() - then).total_seconds() < 5 * \
            60, "run log should save a time to the database"

    def testRunlogSuccess(self):
        script = dedent("""
            import scraperwiki.runlog
            print scraperwiki.runlog.setup()
        """)
        process = Popen(["python", "-c", script],
                        stdout=PIPE, stderr=PIPE, stdin=open("/dev/null"))
        stdout, stderr = process.communicate()

        l = scraperwiki.sqlite.select(
            "time, run_id, success from _sw_runlog order by time desc limit 1")

        # Check that some record is stored.
        assert l
        # Check that it has saved a success column.
        assert l[0]['success']
        # Check that a run_id has been saved.
        match = re.match(r'^\w{8}-\w{4}-\w{4}-\w{4}-\w{12}', stdout)
        assert match.group() == l[0].get(
            'run_id'), "runlog should save a run_id to the database"
        # Check that the time is relatively recent.
        then = datetime.datetime.strptime(l[0]['time'], '%Y-%m-%d %H:%M:%S.%f')
        assert (datetime.datetime.now() - then).total_seconds() < 5 * 60


class TestSaveGetVar(TestDb):

    def savegetvar(self, var):
        scraperwiki.sqlite.save_var("weird", var)
        self.assertEqual(scraperwiki.sqlite.get_var("weird"), var)

    def test_string(self):
        self.savegetvar("asdio")

    def test_int(self):
        self.savegetvar(1)

    # def test_list(self):
    #  self.savegetvar([1,2,3,4])

    # def test_dict(self):
    #  self.savegetvar({"abc":"def"})

    def test_date(self):
        date1 = datetime.datetime.now()
        date2 = datetime.date.today()
        scraperwiki.sqlite.save_var("weird", date1)
        self.assertEqual(scraperwiki.sqlite.get_var("weird"), unicode(date1))
        scraperwiki.sqlite.save_var("weird", date2)
        self.assertEqual(scraperwiki.sqlite.get_var("weird"), unicode(date2))

    def test_save_multiple_values(self):
        scraperwiki.sqlite.save_var('foo', 'hello')
        scraperwiki.sqlite.save_var('bar', 'goodbye')

        self.assertEqual('hello', scraperwiki.sqlite.get_var('foo'))
        self.assertEqual('goodbye', scraperwiki.sqlite.get_var('bar'))


class TestGetNonexistantVar(TestDb):

    def test_get(self):
        self.assertIsNone(scraperwiki.sqlite.get_var('meatball'))


class TestSaveVar(TestDb):

    def setUp(self):
        super(TestSaveVar, self).setUp()
        scraperwiki.sqlite.save_var("birthday", "November 30, 1888")
        connection = sqlite3.connect(self.DBNAME)
        self.cursor = connection.cursor()

    def test_insert(self):
        self.cursor.execute("SELECT name, value_blob, type FROM `swvariables`")
        observed = self.cursor.fetchall()
        expected = [("birthday", "November 30, 1888", "text",)]
        self.assertEqual(observed, expected)


class SaveAndCheck(TestDb):

    def save_and_check(self, dataIn, tableIn, dataOut, tableOut=None, twice=True):
        if tableOut == None:
            tableOut = '[' + tableIn + ']'

        # Insert
        scraperwiki.sqlite.save([], dataIn, tableIn)
        scraperwiki.sqlite.flush()

        # Observe with pysqlite
        connection = sqlite3.connect(self.DBNAME)
        cursor = connection.cursor()
        cursor.execute("SELECT * FROM %s" % tableOut)
        observed1 = cursor.fetchall()
        connection.close()

        if twice:
            # Observe with DumpTruck
            observed2 = scraperwiki.sqlite.select('* FROM %s' % tableOut)

            # Check
            expected1 = dataOut
            expected2 = [dataIn] if type(dataIn) == dict else dataIn

            self.assertListEqual(observed1, expected1)<commit_msg>Dumptruck now fetches BLOBs into a buffer<commit_after>#!/usr/bin/env python
from unittest import TestCase, main
from json import loads, dumps
from subprocess import Popen, PIPE
from textwrap import dedent
import sqlite3
import os
import shutil
import datetime
import urllib2
import re

# This library
import scraperwiki


class TestDb(TestCase):
    DBNAME = 'scraperwiki.sqlite'

    def setUp(self):
        self.cleanUp()
        scraperwiki.sqlite._connect(self.DBNAME)

    def tearDown(self):
        self.cleanUp()

    def cleanUp(self):
        "Clean up temporary files, then reinitialize."
        if self.DBNAME != ':memory:':
            try:
                os.remove(self.DBNAME)
            except OSError:
                pass


class TestException(TestDb):

    def testExceptionSaved(self):
        script = dedent("""
            import scraperwiki.runlog
            print scraperwiki.runlog.setup()
            raise ValueError
        """)
        process = Popen(["python", "-c", script],
                        stdout=PIPE, stderr=PIPE, stdin=open("/dev/null"))
        stdout, stderr = process.communicate()

        assert 'Traceback' in stderr, "stderr should contain the original Python traceback"
        match = re.match(r'^\w{8}-\w{4}-\w{4}-\w{4}-\w{12}', stdout)
        assert match, "runlog.setup() should return a run_id"

        l = scraperwiki.sqlite.select(
            "exception_type, run_id, time from _sw_runlog order by time desc limit 1")

        # Check that some record is stored.
        assert l
        # Check that the exception name appears.
        assert 'ValueError' in l[0][
            'exception_type'], "runlog should save exception types to the database"
        # Check that the run_id from earlier has been saved.
        assert match.group() == l[0].get(
            'run_id'), "runlog should save a run_id to the database"
        # Check that the time recorded is relatively recent.
        time_str = l[0]['time']
        then = datetime.datetime.strptime(time_str, '%Y-%m-%d %H:%M:%S.%f')
        assert (datetime.datetime.now() - then).total_seconds() < 5 * \
            60, "run log should save a time to the database"

    def testRunlogSuccess(self):
        script = dedent("""
            import scraperwiki.runlog
            print scraperwiki.runlog.setup()
        """)
        process = Popen(["python", "-c", script],
                        stdout=PIPE, stderr=PIPE, stdin=open("/dev/null"))
        stdout, stderr = process.communicate()

        l = scraperwiki.sqlite.select(
            "time, run_id, success from _sw_runlog order by time desc limit 1")

        # Check that some record is stored.
        assert l
        # Check that it has saved a success column.
        assert l[0]['success']
        # Check that a run_id has been saved.
        match = re.match(r'^\w{8}-\w{4}-\w{4}-\w{4}-\w{12}', stdout)
        assert match.group() == l[0].get(
            'run_id'), "runlog should save a run_id to the database"
        # Check that the time is relatively recent.
        then = datetime.datetime.strptime(l[0]['time'], '%Y-%m-%d %H:%M:%S.%f')
        assert (datetime.datetime.now() - then).total_seconds() < 5 * 60


class TestSaveGetVar(TestDb):

    def savegetvar(self, var):
        scraperwiki.sqlite.save_var("weird", var)
        self.assertEqual(scraperwiki.sqlite.get_var("weird"), var)

    def test_string(self):
        self.savegetvar("asdio")

    def test_int(self):
        self.savegetvar(1)

    # def test_list(self):
    #  self.savegetvar([1,2,3,4])

    # def test_dict(self):
    #  self.savegetvar({"abc":"def"})

    def test_date(self):
        date1 = datetime.datetime.now()
        date2 = datetime.date.today()
        scraperwiki.sqlite.save_var("weird", date1)
        self.assertEqual(scraperwiki.sqlite.get_var("weird"), unicode(date1))
        scraperwiki.sqlite.save_var("weird", date2)
        self.assertEqual(scraperwiki.sqlite.get_var("weird"), unicode(date2))

    def test_save_multiple_values(self):
        scraperwiki.sqlite.save_var('foo', 'hello')
        scraperwiki.sqlite.save_var('bar', 'goodbye')

        self.assertEqual('hello', scraperwiki.sqlite.get_var('foo'))
        self.assertEqual('goodbye', scraperwiki.sqlite.get_var('bar'))


class TestGetNonexistantVar(TestDb):

    def test_get(self):
        self.assertIsNone(scraperwiki.sqlite.get_var('meatball'))


class TestSaveVar(TestDb):

    def setUp(self):
        super(TestSaveVar, self).setUp()
        scraperwiki.sqlite.save_var("birthday", "November 30, 1888")
        connection = sqlite3.connect(self.DBNAME)
        self.cursor = connection.cursor()

    def test_insert(self):
        self.cursor.execute("SELECT name, value_blob, type FROM `swvariables`")
        observed = self.cursor.fetchall()
        expected = [("birthday", buffer("November 30, 1888"), "text",)]
        self.assertEqual(observed, expected)


class SaveAndCheck(TestDb):

    def save_and_check(self, dataIn, tableIn, dataOut, tableOut=None, twice=True):
        if tableOut == None:
            tableOut = '[' + tableIn + ']'

        # Insert
        scraperwiki.sqlite.save([], dataIn, tableIn)
        scraperwiki.sqlite.flush()

        # Observe with pysqlite
        connection = sqlite3.connect(self.DBNAME)
        cursor = connection.cursor()
        cursor.execute("SELECT * FROM %s" % tableOut)
        observed1 = cursor.fetchall()
        connection.close()

        if twice:
            # Observe with DumpTruck
            observed2 = scraperwiki.sqlite.select('* FROM %s' % tableOut)

            # Check
            expected1 = dataOut
            expected2 = [dataIn] if type(dataIn) == dict else dataIn

            self.assertListEqual(observed1, expected1)
{'fraction_non_alphanumeric': 0.12387231089521165, 'fraction_numerical': 0.010235947258848022, 'mean_word_length': 2.503190519598906, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2902424', 'n_tokens_mistral': 3640, 'n_tokens_neox': 3494, 'n_words': 845}