<commit_before>"""Bert Preprocessor."""

from tqdm import tqdm

from . import units
from .chain_transform import chain_transform
from matchzoo import DataPack
from matchzoo.engine.base_preprocessor import BasePreprocessor
from .build_vocab_unit import built_bert_vocab_unit
from .build_unit_from_data_pack import build_unit_from_data_pack

tqdm.pandas()


class BertPreprocessor(BasePreprocessor):
    """Bert-base Model preprocessor."""

    def __init__(self, fixed_length_left: int = 30,
                 fixed_length_right: int = 30,
                 filter_mode: str = 'df',
                 filter_low_freq: float = 2,
                 filter_high_freq: float = float('inf'),
                 remove_stop_words: bool = False,
                 lower_case: bool=True,
                 chinese_version: bool=True,
                 bert_vocab_path: str="bert_resources/vocab.txt"):
        """
        Bert-base Model preprocessor.

        TODO: doc here.


        Example:
            >>> import matchzoo as mz
            >>> train_data = mz.datasets.toy.load_data()
            >>> test_data = mz.datasets.toy.load_data(stage='test')
            >>> # the argument 'bert_vocab_path' must feed the bert vocab path
            >>> bert_preprocessor = mz.preprocessors.BertPreprocessor(bert_vocab_path='../datasets/bert_resources/uncased_vocab.txt')
            >>> train_data_processed = bert_preprocessor.fit_transform(train_data)
            >>> test_data_processed = bert_preprocessor.transform(test_data)

        """
        super().__init__()
        self._fixed_length_left = fixed_length_left
        self._fixed_length_right = fixed_length_right
        self._bert_vocab_path = bert_vocab_path
        self._left_fixedlength_unit = units.FixedLength(
            self._fixed_length_left,
            pad_mode='post'
        )
        self._right_fixedlength_unit = units.FixedLength(
            self._fixed_length_right,
            pad_mode='post'
        )
        self._filter_unit = units.FrequencyFilter(
            low=filter_low_freq,
            high=filter_high_freq,
            mode=filter_mode
        )
        self._units = self._default_units()
        self._vocab_unit = built_bert_vocab_unit(self._bert_vocab_path)
<commit_msg>fix the vocab_path and pass the quick test<commit_after>"""Bert Preprocessor."""

from tqdm import tqdm

from . import units
from .chain_transform import chain_transform
from matchzoo import DataPack
from matchzoo.engine.base_preprocessor import BasePreprocessor
from .build_vocab_unit import built_bert_vocab_unit
from .build_unit_from_data_pack import build_unit_from_data_pack

tqdm.pandas()


class BertPreprocessor(BasePreprocessor):
    """Bert-base Model preprocessor."""

    def __init__(self, fixed_length_left: int = 30,
                 fixed_length_right: int = 30,
                 filter_mode: str = 'df',
                 filter_low_freq: float = 2,
                 filter_high_freq: float = float('inf'),
                 remove_stop_words: bool = False,
                 lower_case: bool=True,
                 chinese_version: bool=True,
                 bert_vocab_path: str="bert_resources/vocab.txt"):
        """
        Bert-base Model preprocessor.

        TODO: doc here.


        Example:
            >>> import matchzoo as mz
            >>> train_data = mz.datasets.toy.load_data()
            >>> test_data = mz.datasets.toy.load_data(stage='test')
            >>> # the argument 'bert_vocab_path' must feed the bert vocab path
            >>> bert_preprocessor = mz.preprocessors.BertPreprocessor(bert_vocab_path='matchzoo/datasets/bert_resources/uncased_vocab.txt')
            >>> train_data_processed = bert_preprocessor.fit_transform(train_data)
            >>> test_data_processed = bert_preprocessor.transform(test_data)

        """
        super().__init__()
        self._fixed_length_left = fixed_length_left
        self._fixed_length_right = fixed_length_right
        self._bert_vocab_path = bert_vocab_path
        self._left_fixedlength_unit = units.FixedLength(
            self._fixed_length_left,
            pad_mode='post'
        )
        self._right_fixedlength_unit = units.FixedLength(
            self._fixed_length_right,
            pad_mode='post'
        )
        self._filter_unit = units.FrequencyFilter(
            low=filter_low_freq,
            high=filter_high_freq,
            mode=filter_mode
        )
        self._units = self._default_units()
        self._vocab_unit = built_bert_vocab_unit(self._bert_vocab_path)

{'fraction_non_alphanumeric': 0.08430296377607026, 'fraction_numerical': 0.0021953896816684962, 'mean_word_length': 2.472560975609756, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5314700', 'n_tokens_mistral': 1386, 'n_tokens_neox': 1353, 'n_words': 237}