<commit_before>import os
import re
import traceback
import sys
import urllib
from urllib.parse import urlencode
import httplib2
import json
import yaml
import polib
from geojson import loads as geojson_loads
from sqlalchemy.exc import NoSuchTableError, OperationalError, ProgrammingError
from lingua.extractors import Extractor, Message

from c2cgeoportal_geoportal import init_dbsessions
from c2cgeoportal_commons.config import config
from c2cgeoportal_geoportal.lib.bashcolor import RED, colorize

import logging
log = logging.getLogger(__name__)


class LuxembourgExtractor(Extractor):  # pragma: no cover
    """
    ESRI legend extractor
    """
    extensions = [".ini"]

    # Get the keys existing in po file and not in pot file and add them
    def _get_missing_keys(self, po_name):
        po_path = "geoportal/geoportailv3_geoportal/locale/" + po_name
        pot = self.messages
        po = None
        keys = []
        if os.path.exists(po_path):
            po = polib.pofile(po_path, encoding="utf-8")
        else:
            print("File not found")
            return keys

        for po_entry in po:
            found = False
            for pot_entry in pot:
                if pot_entry.msgid == po_entry.msgid:
                    found = True
                    break
            if not found:
                keys.append(po_entry)
        return keys

    def __init__(self):
        super().__init__()
        log.info(f'entering into {self.__class__} lux extractor')
        if os.path.exists("geoportal/config.yaml"):
            config.init("geoportal/config.yaml")
            self.config = config.get_config()
        else:
            self.config = None
        if os.path.exists("project.yaml"):
            with open("project.yaml") as f:
                self.package = yaml.safe_load(f)
        else:
            self.package = None
        self.env = None
        self.messages = []
        self.fields = set()
        self.TIMEOUT = 15

    def __call__(self, filename, options, fileobj=None, lineno=0):
        print('Entering into %s extractor' % self.__class__)

        del fileobj, lineno

        try:
            # initialize DB connections in a way similar to c2cgeoportal_geoportal.lib.lingua_extractor
            settings = config.get_config()

            class R:
                settings = None

            class C:
                registry = R()

            config_ = C()
            config_.registry.settings = settings
            init_dbsessions(settings, config_)
            try:
                self._extract_messages()
                file = "geoportailv3_geoportal-legends.pot"
                if str(self.__class__).index('LuxembourgTooltipsExtractor') > 0:
                    file = "geoportailv3_geoportal-tooltips.pot"
                for attribute in self._get_missing_keys(file):
                    self._insert_attribute(
                        attribute.msgid,
                        (attribute.comment, ""))
            except ProgrammingError as e:
                print(
                    colorize(
                        "ERROR! The database is probably not up to date "
                        "(should be ignored when happen during the upgrade)",
                        RED,
                    )
                )
                print(colorize(e, RED))
                if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
                    raise

        except NoSuchTableError as e:
            print(
                colorize(
                    "ERROR! The schema didn't seem to exists "
                    "(should be ignored when happen during the deploy)",
                    RED,
                )<commit_msg>use find instead of index<commit_after>import os
import re
import traceback
import sys
import urllib
from urllib.parse import urlencode
import httplib2
import json
import yaml
import polib
from geojson import loads as geojson_loads
from sqlalchemy.exc import NoSuchTableError, OperationalError, ProgrammingError
from lingua.extractors import Extractor, Message

from c2cgeoportal_geoportal import init_dbsessions
from c2cgeoportal_commons.config import config
from c2cgeoportal_geoportal.lib.bashcolor import RED, colorize

import logging
log = logging.getLogger(__name__)


class LuxembourgExtractor(Extractor):  # pragma: no cover
    """
    ESRI legend extractor
    """
    extensions = [".ini"]

    # Get the keys existing in po file and not in pot file and add them
    def _get_missing_keys(self, po_name):
        po_path = "geoportal/geoportailv3_geoportal/locale/" + po_name
        pot = self.messages
        po = None
        keys = []
        if os.path.exists(po_path):
            po = polib.pofile(po_path, encoding="utf-8")
        else:
            print("File not found")
            return keys

        for po_entry in po:
            found = False
            for pot_entry in pot:
                if pot_entry.msgid == po_entry.msgid:
                    found = True
                    break
            if not found:
                keys.append(po_entry)
        return keys

    def __init__(self):
        super().__init__()
        log.info(f'entering into {self.__class__} lux extractor')
        if os.path.exists("geoportal/config.yaml"):
            config.init("geoportal/config.yaml")
            self.config = config.get_config()
        else:
            self.config = None
        if os.path.exists("project.yaml"):
            with open("project.yaml") as f:
                self.package = yaml.safe_load(f)
        else:
            self.package = None
        self.env = None
        self.messages = []
        self.fields = set()
        self.TIMEOUT = 15

    def __call__(self, filename, options, fileobj=None, lineno=0):
        print('Entering into %s extractor' % self.__class__)

        del fileobj, lineno

        try:
            # initialize DB connections in a way similar to c2cgeoportal_geoportal.lib.lingua_extractor
            settings = config.get_config()

            class R:
                settings = None

            class C:
                registry = R()

            config_ = C()
            config_.registry.settings = settings
            init_dbsessions(settings, config_)
            try:
                self._extract_messages()
                file = "geoportailv3_geoportal-legends.pot"
                if str(self.__class__).find('LuxembourgTooltipsExtractor') > 0:
                    file = "geoportailv3_geoportal-tooltips.pot"
                for attribute in self._get_missing_keys(file):
                    self._insert_attribute(
                        attribute.msgid,
                        (attribute.comment, ""))
            except ProgrammingError as e:
                print(
                    colorize(
                        "ERROR! The database is probably not up to date "
                        "(should be ignored when happen during the upgrade)",
                        RED,
                    )
                )
                print(colorize(e, RED))
                if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
                    raise

        except NoSuchTableError as e:
            print(
                colorize(
                    "ERROR! The schema didn't seem to exists "
                    "(should be ignored when happen during the deploy)",
                    RED,
                )
{'fraction_non_alphanumeric': 0.07625184588535373, 'fraction_numerical': 0.004027386226339106, 'mean_word_length': 1.6446574369897053, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2888347', 'n_tokens_mistral': 2063, 'n_tokens_neox': 1945, 'n_words': 540}