<commit_before># -*- encoding: utf-8 -*-

import os
import time

import click
from lxml import etree

from perfectextractor.apps.extractor.base import BaseExtractor
from perfectextractor.apps.extractor.models import Alignment
from perfectextractor.apps.extractor.perfectextractor import PerfectExtractor, PRESENT
from perfectextractor.apps.extractor.posextractor import PoSExtractor
from perfectextractor.apps.extractor.recentpastextractor import RecentPastExtractor
from perfectextractor.apps.extractor.xml_utils import get_sentence_from_element
from perfectextractor.apps.extractor.utils import XML

from .base import BaseEuroparl


class EuroparlExtractor(BaseEuroparl, BaseExtractor):
    def __init__(self, *args, **kwargs):
        self._index = dict()  # save segments indexed by id
        super().__init__(*args, **kwargs)

    def process_file(self, filename):
        """
        Processes a single file.
        """
        t0 = time.time()
        click.echo('Now processing {}...'.format(filename))

        # Parse the current tree (create a iterator over 's' elements)
        s_trees = etree.iterparse(filename, tag='s')

        # Parse the alignment and translation trees
        alignment_trees, translation_trees = self.parse_alignment_trees(filename)

        t1 = time.time()
        click.echo('Finished parsing trees, took {:.3} seconds'.format(t1 - t0))

        # Fetch the results
        results = self.fetch_results(filename, s_trees, alignment_trees, translation_trees)

        click.echo('Finished fetching results, took {:.3} seconds'.format(time.time() - t1))

        return results

    def fetch_results(self, filename, s_trees, alignment_trees, translation_trees):
        """
        Fetches the results for a file
        :param filename: The current filename
        :param s_trees: The XML trees for all 's' elements in the file
        :param alignment_trees: The alignment XML trees, per target language
        :param translation_trees: The translation XML trees, per target language
        :return: All results in a list
        """
        results = list()
        # Loop over all sentences
        for _, s in s_trees:
            if self.sentence_ids and s.get('id') not in self.sentence_ids:
                continue

            result = list()
            result.append(os.path.basename(filename))
            result.append(s.get('id'))
            result.append('')<commit_msg>performance: fixed index memory use<commit_after># -*- encoding: utf-8 -*-

import os
import time

import click
from lxml import etree

from perfectextractor.apps.extractor.base import BaseExtractor
from perfectextractor.apps.extractor.models import Alignment
from perfectextractor.apps.extractor.perfectextractor import PerfectExtractor, PRESENT
from perfectextractor.apps.extractor.posextractor import PoSExtractor
from perfectextractor.apps.extractor.recentpastextractor import RecentPastExtractor
from perfectextractor.apps.extractor.xml_utils import get_sentence_from_element
from perfectextractor.apps.extractor.utils import XML

from .base import BaseEuroparl


class EuroparlExtractor(BaseEuroparl, BaseExtractor):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

    def process_file(self, filename):
        """
        Processes a single file.
        """
        self._index = dict()  # save segments indexed by id
        t0 = time.time()
        click.echo('Now processing {}...'.format(filename))

        # Parse the current tree (create a iterator over 's' elements)
        s_trees = etree.iterparse(filename, tag='s')

        # Parse the alignment and translation trees
        alignment_trees, translation_trees = self.parse_alignment_trees(filename)

        t1 = time.time()
        click.echo('Finished parsing trees, took {:.3} seconds'.format(t1 - t0))

        # Fetch the results
        results = self.fetch_results(filename, s_trees, alignment_trees, translation_trees)

        click.echo('Finished fetching results, took {:.3} seconds'.format(time.time() - t1))

        self._index = dict()  # free index memory
        return results

    def fetch_results(self, filename, s_trees, alignment_trees, translation_trees):
        """
        Fetches the results for a file
        :param filename: The current filename
        :param s_trees: The XML trees for all 's' elements in the file
        :param alignment_trees: The alignment XML trees, per target language
        :param translation_trees: The translation XML trees, per target language
        :return: All results in a list
        """
        results = list()
        # Loop over all sentences
        for _, s in s_trees:
            if self.sentence_ids and s.get('id') not in self.sentence_ids:
                continue

            result = list()
            result.append(os.path.basename(filename))
            result.append(s.get('id'))
            result.append('')
{'fraction_non_alphanumeric': 0.08833063209076175, 'fraction_numerical': 0.0032414910858995136, 'mean_word_length': 3.0073051948051948, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4255581', 'n_tokens_mistral': 1392, 'n_tokens_neox': 1304, 'n_words': 404}