<commit_before>"""
    OCR convert tests.
"""
import os
import shutil
from django.test import TestCase
from django.test.client import Client
from django.contrib.auth.models import User
from django.conf import settings
from django.utils import simplejson

from ocradmin.ocr.tests import testutils
from ocradmin.ocrtasks.models import OcrTask

from ocradmin.ocrplugins import parameters


TESTFILE = "etc/simple.png"

class OcrConvertTest(TestCase):
    fixtures = ["ocrmodels/fixtures/test_fixtures.json"]
    def setUp(self):
        """
            Setup OCR tests.  Creates a test user.
        """
        testutils.symlink_model_fixtures()
        self.testuser = User.objects.create_user("test_user", "<EMAIL>", "test<PASSWORD>")
        self.client = Client()
        self.client.login(username="test_user", password="<PASSWORD>")

    def tearDown(self):
        """
            Cleanup a test.
        """
        self.testuser.delete()


    def test_convert_view(self):
        """
        Test the convert view as a standard GET (no processing.)
        """
        self.assertEqual(self.client.get("/ocr/convert").status_code, 200)

    def test_convert_action_basic(self):
        """
        Test OCRing with minimal parameters.
        """
        self._test_convert_action(parameters.TESTPOST_OCROPUS)        

    def test_convert_action_tess(self):
        """
        Test OCRing with Tesseract as the engine.
        """
        self._test_convert_action(parameters.TESTPOST)        

    def test_convert_action_ocropus(self):
        """
        Test OCRing with OCRopus as the engine.
        """
        self._test_convert_action(parameters.TESTPOST_OCROPUS)        

    def test_convert_action_seg(self):
        """
        Test OCRing with variable segmentation.
        """
        self._test_convert_action(parameters.TESTPOST_OCROPUS_SEG)        

    def test_convert_plain_text(self):
        """
        Fetch results and request plain text back.
        """
        params = {"format": "text"}
        params.update(parameters.TESTPOST)
        r = self._get_convert_response(params=params, headers={"accept":"text/plain"})
        self.assertEqual(r.status_code, 200)
        # check the response is not JSON
        self.assertRaises(ValueError, simplejson.loads, r.content)
        self.assertEqual(r["Content-Type"], "text/plain")

    def test_results_plain_text(self):
        """
        Fetch results and request plain text back.
        """
        params = {"format": "json"}
        params.update(parameters.TESTPOST)
        r = self._get_convert_response(params=params)
        self.assertEqual(r.status_code, 200)
        # check the response IS JSON at this stage
        content = simplejson.loads(r.content)
        r2 = self.client.get("/ocr/results/%s" % content[0]["task_id"], {"format": "text"})
        self.assertEqual(r2.status_code, 200)        
        self.assertRaises(ValueError, simplejson.loads, r2.content)
        self.assertEqual(r2["Content-Type"], "text/plain")

    def test_results_action(self):
        """
        Test fetching task results.  We can't meaningfully do this
        locally, because when testing the celery backend  is bypassed.
        However we can get the view works and returns a stub task.
        """
        import uuid
        r = self.client.get("/ocr/results/%s" % uuid.uuid1())
        self.assert_(r.content, "No content returned")
        content = simplejson.loads(r.content)
        # unknown tasks should come back with status 'PENDING'
        self.assertEqual(content["status"], "PENDING")

    def test_transcript_action(self):
        """
        Test viewing the transcript for a new batch.
        """
        r = self._get_convert_response(parameters.TESTPOST)
        self.assertEqual(r.status_code, 200)
        task = OcrTask.objects.all().order_by("-created_on")[0]
        r = self.client.get("/ocr/transcript/%s/" % task.pk)
        self.assertEqual(r.status_code, 200)

    def test_edit_convert_view(self):
        """
        Test viewing the edit task page.
        """
        r = self._get_convert_response(parameters.TESTPOST)
        self.assertEqual(r.status_code, 200)
        task = OcrTask.objects.all().order_by("-created_on")[0]
        r = self.client.get("/ocr/convert/%s/" % task.pk)
        self.assertEqual(r.status_code, 200)

    def test_edit_convert_action(self):
        """
        Test updating a tasks parameters.
        """
        r = self._get_convert_response(parameters.TESTPOST)
        self.assertEqual(r.status_code, 200)
        task = OcrTask.objects.all().order_by("-created_on")[0]
        r = self.client.post(
            "/ocr/update_task/%s/" % task.pk,
            {"%options.tesseract[1].psegmenter": "SegmentPageBy1CP",},                
        )
        self.assertEqual(r.status_code, 302)

    def _test_convert_action(self, params=None, headers={}):
        """
        Testing actually OCRing a file.
        """
        if params is None:
            params = {}
        r = self._get_convert_response(params, headers) 
        self.assertEqual(r.status_code, 200)

        # check we recieve JSON back
        self.assert_(r.content, "No content returned")
        content = simplejson.loads(r.content)
        #print content
        self.assertEqual(len(content), 1)
        # Note: we'd not normally expect any results here because we're
        # not using the "nohang" parameter, but since tests are executed 
        # locally we will
        self.assertTrue(content[0]["results"] is not None, "Unable to get results")
      
    def _get_convert_response(self, params={}, headers={}):
        """
        Post an image for conversion with the given params, headers.
        """
        tf = open(TESTFILE)
        params["image1"] = tf
        r = self.client.post("/ocr/convert", params, **headers)
        tf.close()
        return r<commit_msg>For some reason closing this handle causes an error on subsequent tests.  Oh well.<commit_after>"""
    OCR convert tests.
"""
import os
import shutil
from django.test import TestCase
from django.test.client import Client
from django.contrib.auth.models import User
from django.conf import settings
from django.utils import simplejson

from ocradmin.ocr.tests import testutils
from ocradmin.ocrtasks.models import OcrTask

from ocradmin.ocrplugins import parameters


TESTFILE = "etc/simple.png"

class OcrConvertTest(TestCase):
    fixtures = ["ocrmodels/fixtures/test_fixtures.json"]
    def setUp(self):
        """
            Setup OCR tests.  Creates a test user.
        """
        testutils.symlink_model_fixtures()
        self.testuser = User.objects.create_user("test_user", "<EMAIL>", "<PASSWORD>")
        self.client = Client()
        self.client.login(username="test_user", password="<PASSWORD>")

    def tearDown(self):
        """
            Cleanup a test.
        """
        self.testuser.delete()


    def test_convert_view(self):
        """
        Test the convert view as a standard GET (no processing.)
        """
        self.assertEqual(self.client.get("/ocr/convert").status_code, 200)

    def test_convert_action_basic(self):
        """
        Test OCRing with minimal parameters.
        """
        self._test_convert_action(parameters.TESTPOST_OCROPUS)        

    def test_convert_action_tess(self):
        """
        Test OCRing with Tesseract as the engine.
        """
        self._test_convert_action(parameters.TESTPOST)        

    def test_convert_action_ocropus(self):
        """
        Test OCRing with OCRopus as the engine.
        """
        self._test_convert_action(parameters.TESTPOST_OCROPUS)        

    def test_convert_action_seg(self):
        """
        Test OCRing with variable segmentation.
        """
        self._test_convert_action(parameters.TESTPOST_OCROPUS_SEG)        

    def test_convert_plain_text(self):
        """
        Fetch results and request plain text back.
        """
        params = {"format": "text"}
        params.update(parameters.TESTPOST)
        r = self._get_convert_response(params=params, headers={"accept":"text/plain"})
        self.assertEqual(r.status_code, 200)
        # check the response is not JSON
        self.assertRaises(ValueError, simplejson.loads, r.content)
        self.assertEqual(r["Content-Type"], "text/plain")

    def test_results_plain_text(self):
        """
        Fetch results and request plain text back.
        """
        params = {"format": "json"}
        params.update(parameters.TESTPOST)
        r = self._get_convert_response(params=params)
        self.assertEqual(r.status_code, 200)
        # check the response IS JSON at this stage
        content = simplejson.loads(r.content)
        r2 = self.client.get("/ocr/results/%s" % content[0]["task_id"], {"format": "text"})
        self.assertEqual(r2.status_code, 200)        
        self.assertRaises(ValueError, simplejson.loads, r2.content)
        self.assertEqual(r2["Content-Type"], "text/plain")

    def test_results_action(self):
        """
        Test fetching task results.  We can't meaningfully do this
        locally, because when testing the celery backend  is bypassed.
        However we can get the view works and returns a stub task.
        """
        import uuid
        r = self.client.get("/ocr/results/%s" % uuid.uuid1())
        self.assert_(r.content, "No content returned")
        content = simplejson.loads(r.content)
        # unknown tasks should come back with status 'PENDING'
        self.assertEqual(content["status"], "PENDING")

    def test_transcript_action(self):
        """
        Test viewing the transcript for a new batch.
        """
        r = self._get_convert_response(parameters.TESTPOST)
        self.assertEqual(r.status_code, 200)
        task = OcrTask.objects.all().order_by("-created_on")[0]
        r = self.client.get("/ocr/transcript/%s/" % task.pk)
        self.assertEqual(r.status_code, 200)

    def test_edit_convert_view(self):
        """
        Test viewing the edit task page.
        """
        r = self._get_convert_response(parameters.TESTPOST)
        self.assertEqual(r.status_code, 200)
        task = OcrTask.objects.all().order_by("-created_on")[0]
        r = self.client.get("/ocr/convert/%s/" % task.pk)
        self.assertEqual(r.status_code, 200)

    def test_edit_convert_action(self):
        """
        Test updating a tasks parameters.
        """
        r = self._get_convert_response(parameters.TESTPOST)
        self.assertEqual(r.status_code, 200)
        task = OcrTask.objects.all().order_by("-created_on")[0]
        r = self.client.post(
            "/ocr/update_task/%s/" % task.pk,
            {"%options.tesseract[1].psegmenter": "SegmentPageBy1CP",},                
        )
        self.assertEqual(r.status_code, 302)

    def _test_convert_action(self, params=None, headers={}):
        """
        Testing actually OCRing a file.
        """
        if params is None:
            params = {}
        r = self._get_convert_response(params, headers) 
        self.assertEqual(r.status_code, 200)

        # check we recieve JSON back
        self.assert_(r.content, "No content returned")
        content = simplejson.loads(r.content)
        #print content
        self.assertEqual(len(content), 1)
        # Note: we'd not normally expect any results here because we're
        # not using the "nohang" parameter, but since tests are executed 
        # locally we will
        self.assertTrue(content[0]["results"] is not None, "Unable to get results")
      
    def _get_convert_response(self, params={}, headers={}):
        """
        Post an image for conversion with the given params, headers.
        """
        tf = open(TESTFILE)
        params["image1"] = tf
        r = self.client.post("/ocr/convert", params, **headers)
        #tf.close()
        return r
{'fraction_non_alphanumeric': 0.11298439859084046, 'fraction_numerical': 0.007884583123636974, 'mean_word_length': 2.415353766828989, 'pattern_counts': {'":': 10, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3379011', 'n_tokens_mistral': 3495, 'n_tokens_neox': 3404, 'n_words': 796}