<commit_before>import hashlib
import importlib.util
import json
import logging
import os
import shutil
import tempfile
from unittest import skipIf

from dateutil.relativedelta import relativedelta
from django.test import TestCase
from django.utils import timezone

from mainapp.models import Body, LegislativeTerm, Organization, Person, OrganizationMembership, \
    Meeting, AgendaItem, \
    Paper, Consultation, Location, File

gi_not_available = importlib.util.find_spec("gi") is None
if not gi_not_available:
    # Those two require importing gi
    from importer.oparl_helper import default_options
    from importer.oparl_import import OParlImport
    from importer.oparl_resolve import OParlResolver

logger = logging.getLogger(__name__)


@skipIf(gi_not_available, "gi is not available")
class TestImporter(TestCase):
    dummy_data = "testdata/oparl"
    fake_cache = "testdata/fake_cache"
    base_timestamp = timezone.now().astimezone().replace(microsecond=0)
    new_timestamp = None
    delete = False
    tables = [Body, LegislativeTerm, Organization, Person, OrganizationMembership, Meeting, AgendaItem,
              Paper, Consultation, Location]
    entrypoint = "https://oparl.example.org/"
    tempdir = None

    resolver = OParlResolver(entrypoint, fake_cache, True)

    fixtures = ["cologne-pois-test"]

    @classmethod
    def setUpClass(cls):
        super().setUpClass()
        cls.tempdir = tempfile.mkdtemp()
        cls.options = cls.build_options()

    @classmethod
    def tearDownClass(cls):
        shutil.rmtree(cls.tempdir)

    def sha1(self, data):
        return hashlib.sha1(data.encode("utf-8")).hexdigest()

    def load(self, name):
        with open(os.path.join(self.dummy_data, name)) as f:
            return self.manipulate(json.load(f), self.new_timestamp)
<commit_msg>Fix non-gi tests<commit_after>import hashlib
import importlib.util
import json
import logging
import os
import shutil
import tempfile
from unittest import skipIf

from dateutil.relativedelta import relativedelta
from django.test import TestCase
from django.utils import timezone

from mainapp.models import Body, LegislativeTerm, Organization, Person, OrganizationMembership, \
    Meeting, AgendaItem, \
    Paper, Consultation, Location, File

gi_not_available = importlib.util.find_spec("gi") is None
if not gi_not_available:
    # Those two require importing gi
    from importer.oparl_helper import default_options
    from importer.oparl_import import OParlImport
    from importer.oparl_resolve import OParlResolver

logger = logging.getLogger(__name__)


@skipIf(gi_not_available, "gi is not available")
class TestImporter(TestCase):
    dummy_data = "testdata/oparl"
    fake_cache = "testdata/fake_cache"
    base_timestamp = timezone.now().astimezone().replace(microsecond=0)
    new_timestamp = None
    delete = False
    tables = [Body, LegislativeTerm, Organization, Person, OrganizationMembership, Meeting, AgendaItem,
              Paper, Consultation, Location]
    entrypoint = "https://oparl.example.org/"
    tempdir = None

    resolver = None  # Initializing here will lead to an import error without gi

    fixtures = ["cologne-pois-test"]

    @classmethod
    def setUpClass(cls):
        super().setUpClass()
        cls.tempdir = tempfile.mkdtemp()
        cls.options = cls.build_options()
        cls.resolver = OParlResolver(cls.entrypoint, cls.fake_cache, True)

    @classmethod
    def tearDownClass(cls):
        shutil.rmtree(cls.tempdir)

    def sha1(self, data):
        return hashlib.sha1(data.encode("utf-8")).hexdigest()

    def load(self, name):
        with open(os.path.join(self.dummy_data, name)) as f:
            return self.manipulate(json.load(f), self.new_timestamp)

{'fraction_non_alphanumeric': 0.08920940170940171, 'fraction_numerical': 0.002136752136752137, 'mean_word_length': 3.8763020833333335, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7224149', 'n_tokens_mistral': 1128, 'n_tokens_neox': 1114, 'n_words': 286}