<commit_before>#!/usr/bin/env python3
"""Corpus reader for the Hungarian Webcorpus."""

from __future__ import absolute_import, division, print_function
from html.parser import HTMLParser  # Needs future, too
import io
import itertools
import re
import tarfile

from emLam.corpus.corpus_base import RawCorpus
from emLam.utils import openall


class Webcorpus(RawCorpus):
    NAME = 'hu_webcorpus'
    rename_p = re.compile(r'\.tar(\.gz)$')

    def __init__(self, compressed=True, max_entities=0.2):
        self.compressed = compressed
        self.max_entities = max_entities
        self.html_parser = HTMLParser()

    def files_to_streams(self, input_file, output_file):
        """
        Reads input_file according to the corpus format (compressed / not). In
        the former case, modifies the output_file name so that the '.tar' part
        is not included in it.
        """
        if self.compressed:
            output_file = self.rename_p.sub(r'\1', output_file)
            input_stream = self.enumerate_tar
        else:
            input_stream = self.enumerate_file
        with openall(output_file, 'wt', encoding='utf-8') as outf:
            inf = itertools.chain.from_iterable(input_stream(input_file))
            yield self.__read_sentence(inf), outf

    def __read_sentence(self, input_stream):
        """Returns a sentence a time, cleaned of HTML entities."""
        for line in input_stream:
            if line.startswith(u'<s>'):
                text = line[3:]
                amps = text.count(u'&')
                if amps > 0:
                    text = self.html_parser.unescape(text)
                    entities = text.count(u'&') - amps
                    if entities / float(len(text)) > self.max_entities:
                        # Skip sentence if too many entities (i.e. foreign script)
                        continue
                yield text

    @classmethod
    def child_parser(cls, subparsers):
        parser = subparsers.add_parser('hu_webcorpus', help='Hungarian Webcorpus')
        parser.add_argument('--uncompressed', '-u', action='store_false',
                            dest='compressed',
                            help='the source directory contains the uncompressed '
                                 'corpus (1 file per document). Not recommended; '
                                 'by default, the preprocessor expects '
                                 'tar.gz files.')
        parser.add_argument('--max-entities', '-m', type=float, default=0.2,
                            help='the HTML entity / characters ratio above '
                                 'which a sentence is discarded.')<commit_msg>Webcorpus to use the instream / outstream infrastructure.<commit_after>#!/usr/bin/env python3
"""Corpus reader for the Hungarian Webcorpus."""

from __future__ import absolute_import, division, print_function
from contextlib import contextmanager
from html.parser import HTMLParser  # Needs future, too
import io
import itertools
import re
import tarfile

from emLam.corpus.corpus_base import RawCorpus
from emLam.utils import openall


class Webcorpus(RawCorpus):
    NAME = 'hu_webcorpus'
    rename_p = re.compile(r'\.tar(\.gz)$')

    def __init__(self, compressed=True, max_entities=0.2):
        self.compressed = compressed
        self.max_entities = max_entities
        self.html_parser = HTMLParser()

    @contextmanager
    def instream(self, input_file):
        if self.compressed:
            input_stream = self.enumerate_tar
        else:
            input_stream = self.enumerate_file
        inf = itertools.chain.from_iterable(input_stream(input_file))
        yield self.__read_sentence(inf)

    def outstream(self, output_file):
        """Removes the 'tar' from the name of the output file, if compressed."""
        if self.compressed:
            output_file = self.rename_p.sub(r'\1', output_file)
        return super(Webcorpus, self).outstream(output_file)

#    def files_to_streams(self, input_file, output_file):
#        """
#        Reads input_file according to the corpus format (compressed / not). In
#        the former case, modifies the output_file name so that the '.tar' part
#        is not included in it.
#        """
#        if self.compressed:
#            output_file = self.rename_p.sub(r'\1', output_file)
#            input_stream = self.enumerate_tar
#        else:
#            input_stream = self.enumerate_file
#        with openall(output_file, 'wt', encoding='utf-8') as outf:
#            inf = itertools.chain.from_iterable(input_stream(input_file))
#            yield self.__read_sentence(inf), outf

    def __read_sentence(self, input_stream):
        """Returns a sentence a time, cleaned of HTML entities."""
        for line in input_stream:
            if line.startswith(u'<s>'):
                text = line[3:]
                amps = text.count(u'&')
                if amps > 0:
                    text = self.html_parser.unescape(text)
                    entities = text.count(u'&') - amps
                    if entities / float(len(text)) > self.max_entities:
                        # Skip sentence if too many entities (i.e. foreign script)
                        continue
                yield text

    @classmethod
    def child_parser(cls, subparsers):
        parser = subparsers.add_parser('hu_webcorpus', help='Hungarian Webcorpus')
        parser.add_argument('--uncompressed', '-u', action='store_false',
                            dest='compressed',
                            help='the source directory contains the uncompressed '
                                 'corpus (1 file per document). Not recommended; '
                                 'by default, the preprocessor expects '
                                 'tar.gz files.')
        parser.add_argument('--max-entities', '-m', type=float, default=0.2,
                            help='the HTML entity / characters ratio above '
                                 'which a sentence is discarded.')
{'fraction_non_alphanumeric': 0.09507217521154804, 'fraction_numerical': 0.003484320557491289, 'mean_word_length': 2.029145728643216, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4711234', 'n_tokens_mistral': 1671, 'n_tokens_neox': 1636, 'n_words': 443}