<commit_before># coding=utf-8
import os

from cl import settings
from cl.audio.models import Audio
from cl.lib.scrape_helpers import get_extension
from cl.lib.test_helpers import IndexedSolrTestCase
from cl.scrapers.DupChecker import DupChecker
from cl.scrapers.models import UrlHash, ErrorLog
from cl.scrapers.management.commands import (
    cl_report_scrape_status, cl_scrape_opinions, cl_scrape_oral_arguments
)
from cl.scrapers.tasks import (
    extract_from_txt, extract_doc_content, extract_by_ocr, process_audio_file
)
from cl.scrapers.test_assets import test_opinion_scraper, test_oral_arg_scraper
from cl.search.models import Court, Opinion

from celery.task.sets import subtask
from datetime import timedelta
from django.test import TestCase
from django.utils.timezone import now


class IngestionTest(IndexedSolrTestCase):
    fixtures = ['test_court.json']

    def test_ingest_opinions(self):
        """Can we successfully ingest opinions at a high level?"""
        site = test_opinion_scraper.Site()
        site.method = "LOCAL"
        parsed_site = site.parse()
        cl_scrape_opinions.Command().scrape_court(parsed_site, full_crawl=True)

        self.assertTrue(False, msg="Need to check the DB for content here.")

    def test_ingest_oral_arguments(self):
        """Can we successfully ingest oral arguments at a high level?"""
        site = test_oral_arg_scraper.Site()
        site.method = "LOCAL"
        parsed_site = site.parse()
        cl_scrape_oral_arguments.Command().scrape_court(
            parsed_site,
            full_crawl=True
        )

        # There should now be two items in the database.
        audio_files = Audio.objects.all()
        self.assertEqual(2, audio_files.count())

    def test_parsing_xml_opinion_site_to_site_object(self):
        """Does a basic parse of a site reveal the right number of items?"""
        site = test_opinion_scraper.Site().parse()
        self.assertEqual(len(site.case_names), 6)

    def test_parsing_xml_oral_arg_site_to_site_object(self):
        """Does a basic parse of an oral arg site work?"""
        site = test_oral_arg_scraper.Site().parse()
        self.assertEqual(len(site.case_names), 2)

    def test_content_extraction(self):
        """Do all of the supported mimetypes get extracted to text
        successfully, including OCR?"""<commit_msg>test_ingest_opinions changed to test for 6 test opinions being loaded into the DB<commit_after># coding=utf-8
import os

from cl import settings
from cl.audio.models import Audio
from cl.lib.scrape_helpers import get_extension
from cl.lib.test_helpers import IndexedSolrTestCase
from cl.scrapers.DupChecker import DupChecker
from cl.scrapers.models import UrlHash, ErrorLog
from cl.scrapers.management.commands import (
    cl_report_scrape_status, cl_scrape_opinions, cl_scrape_oral_arguments
)
from cl.scrapers.tasks import (
    extract_from_txt, extract_doc_content, extract_by_ocr, process_audio_file
)
from cl.scrapers.test_assets import test_opinion_scraper, test_oral_arg_scraper
from cl.search.models import Court, Opinion

from celery.task.sets import subtask
from datetime import timedelta
from django.test import TestCase
from django.utils.timezone import now


class IngestionTest(IndexedSolrTestCase):
    fixtures = ['test_court.json']

    def test_ingest_opinions(self):
        """Can we successfully ingest opinions at a high level?"""
        site = test_opinion_scraper.Site()
        site.method = "LOCAL"
        parsed_site = site.parse()
        cl_scrape_opinions.Command().scrape_court(parsed_site, full_crawl=True)

        opinions = Opinion.objects.all()
        self.assertTrue(opinions.count() == 6, 'Should have 6 test opinions.')

    def test_ingest_oral_arguments(self):
        """Can we successfully ingest oral arguments at a high level?"""
        site = test_oral_arg_scraper.Site()
        site.method = "LOCAL"
        parsed_site = site.parse()
        cl_scrape_oral_arguments.Command().scrape_court(
            parsed_site,
            full_crawl=True
        )

        # There should now be two items in the database.
        audio_files = Audio.objects.all()
        self.assertEqual(2, audio_files.count())

    def test_parsing_xml_opinion_site_to_site_object(self):
        """Does a basic parse of a site reveal the right number of items?"""
        site = test_opinion_scraper.Site().parse()
        self.assertEqual(len(site.case_names), 6)

    def test_parsing_xml_oral_arg_site_to_site_object(self):
        """Does a basic parse of an oral arg site work?"""
        site = test_oral_arg_scraper.Site().parse()
        self.assertEqual(len(site.case_names), 2)

    def test_content_extraction(self):
        """Do all of the supported mimetypes get extracted to text
        successfully, including OCR?"""
{'fraction_non_alphanumeric': 0.08151835718730553, 'fraction_numerical': 0.0022816842978635138, 'mean_word_length': 3.690661478599222, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1029109', 'n_tokens_mistral': 1515, 'n_tokens_neox': 1452, 'n_words': 379}