<commit_before>package eu.modernmt.processing;

import eu.modernmt.lang.Language;
import eu.modernmt.lang.LanguageDirection;
import eu.modernmt.model.Sentence;
import eu.modernmt.model.Tag;
import eu.modernmt.model.Word;
import eu.modernmt.model.XMLTag;
import org.apache.commons.io.IOUtils;
import org.junit.Test;

import java.io.IOException;

import static org.junit.Assert.*;

public class PreprocessorTest {

    private static final LanguageDirection language = new LanguageDirection(Language.ENGLISH, Language.ITALIAN);

    private static Sentence process(String text) throws ProcessingException {
        Preprocessor preprocessor = null;

        try {
            preprocessor = new Preprocessor();
            return preprocessor.process(language, text);
        } catch (IOException e) {
            throw new ProcessingException(e);
        } finally {
            IOUtils.closeQuietly(preprocessor);
        }
    }

    @Test
    public void testCommonSentence() throws ProcessingException {
        String text = "Hello world!";
        Sentence sentence = process(text);

        assertEquals(text, sentence.toString(true, false));
        assertEquals("Hello world!", sentence.toString(false, false));
        assertFalse(sentence.hasTags());

        assertArrayEquals(new Word[]{
                new Word("Hello", "Hello", null," "),
                new Word("world", "world", " ",null),
                new Word("!", "!",  null,null),
        }, sentence.getWords());
    }

    @Test
    public void testInitialTagWithSpace() throws ProcessingException {
        String text = "<a> Hello world!";
        Sentence sentence = process(text);

        assertEquals(text, sentence.toString(true, false));
        assertEquals("Hello world!", sentence.toString(false, false));
        assertTrue(sentence.hasTags());

        assertArrayEquals(new Word[]{
                new Word("Hello", "Hello", " "," "),
                new Word("world", "world", " ", null),
                new Word("!", "!", null,null),
        }, sentence.getWords());
        assertArrayEquals(new Tag[]{
                XMLTag.fromText("<a>", null, " ", 0)
        }, sentence.getTags());
    }

    @Test
    public void testStrippedSentenceWithSpaceAfterTag() throws ProcessingException {
        String text = "Hello<a> world!";
        Sentence sentence = process(text);

        assertEquals(text, sentence.toString(true, false));
        assertEquals("Hello world!", sentence.toString(false, false));
        assertTrue(sentence.hasTags());

        assertArrayEquals(new Word[]{
                new Word("Hello", "Hello", null,null),
                new Word("world", "world", " ", null),
                new Word("!", "!", null, null),
        }, sentence.getWords());
        assertArrayEquals(new Tag[]{
                XMLTag.fromText("<a>", null, " ", 1)
        }, sentence.getTags());
    }

    @Test
    public void testStrippedSentenceWithSpacesBetweenTags() throws ProcessingException {
        String text = "Hello<a> <b>world!";
        Sentence sentence = process(text);

        assertEquals(text, sentence.toString(true, false));
        assertEquals("Hello world!", sentence.toString(false, false));
        assertTrue(sentence.hasTags());

        assertArrayEquals(new Word[]{
                new Word("Hello", "Hello", null, null),
                new Word("world", "world", null, null),
                new Word("!", "!", null, null),
        }, sentence.getWords());
        assertArrayEquals(new Tag[]{
                XMLTag.fromText("<a>", null, " ", 1),
                XMLTag.fromText("<b>", " ", null, 1)
        }, sentence.getTags());
    }

}<commit_msg>added tests<commit_after>package eu.modernmt.processing;

import eu.modernmt.lang.Language;
import eu.modernmt.lang.LanguageDirection;
import eu.modernmt.model.Sentence;
import eu.modernmt.model.Tag;
import eu.modernmt.model.Word;
import eu.modernmt.model.XMLTag;
import org.apache.commons.io.IOUtils;
import org.junit.Test;

import java.io.IOException;

import static org.junit.Assert.*;

public class PreprocessorTest {

    private static final LanguageDirection language = new LanguageDirection(Language.ENGLISH, Language.ITALIAN);

    private static Sentence process(String text) throws ProcessingException {
        Preprocessor preprocessor = null;

        try {
            preprocessor = new Preprocessor();
            return preprocessor.process(language, text);
        } catch (IOException e) {
            throw new ProcessingException(e);
        } finally {
            IOUtils.closeQuietly(preprocessor);
        }
    }

    @Test
    public void testCommonSentence() throws ProcessingException {
        String text = "Hello world!";
        Sentence sentence = process(text);

        assertEquals(text, sentence.toString(true, false));
        assertEquals("Hello world!", sentence.toString(false, false));
        assertFalse(sentence.hasTags());

        assertArrayEquals(new Word[]{
                new Word("Hello", "Hello", null," "),
                new Word("world", "world", " ",null),
                new Word("!", "!",  null,null),
        }, sentence.getWords());
    }

    @Test
    public void testInitialTagWithSpace() throws ProcessingException {
        String text = "<a> Hello world!";
        Sentence sentence = process(text);

        assertEquals(text, sentence.toString(true, false));
        assertEquals("Hello world!", sentence.toString(false, false));
        assertTrue(sentence.hasTags());

        assertArrayEquals(new Word[]{
                new Word("Hello", "Hello", " "," "),
                new Word("world", "world", " ", null),
                new Word("!", "!", null,null),
        }, sentence.getWords());
        assertArrayEquals(new Tag[]{
                XMLTag.fromText("<a>", null, " ", 0)
        }, sentence.getTags());
    }

    @Test
    public void testStrippedSentenceWithSpaceAfterTag() throws ProcessingException {
        String text = "Hello<a> world!";
        Sentence sentence = process(text);

        assertEquals(text, sentence.toString(true, false));
        assertEquals("Hello world!", sentence.toString(false, false));
        assertTrue(sentence.hasTags());

        assertArrayEquals(new Word[]{
                new Word("Hello", "Hello", null,null),
                new Word("world", "world", " ", null),
                new Word("!", "!", null, null),
        }, sentence.getWords());
        assertArrayEquals(new Tag[]{
                XMLTag.fromText("<a>", null, " ", 1)
        }, sentence.getTags());
    }

    @Test
    public void testStrippedSentenceWithSpacesBetweenTags() throws ProcessingException {
        String text = "Hello<a> <b>world!";
        Sentence sentence = process(text);

        assertEquals(text, sentence.toString(true, false));
        assertEquals("Hello world!", sentence.toString(false, false));
        assertTrue(sentence.hasTags());

        assertArrayEquals(new Word[]{
                new Word("Hello", "Hello", null, null),
                new Word("world", "world", null, null),
                new Word("!", "!", null, null),
        }, sentence.getWords());
        assertArrayEquals(new Tag[]{
                XMLTag.fromText("<a>", null, " ", 1),
                XMLTag.fromText("<b>", " ", null, 1)
        }, sentence.getTags());
    }

    @Test
    public void testRequiredSpaceTrue() throws ProcessingException {
        String text = "Hello<a>guys";
        Sentence sentence = process(text);

        assertEquals(text, sentence.toString(true, false));
        assertEquals("Hello guys", sentence.toString(false, false));
        assertTrue(sentence.hasTags());

        assertArrayEquals(new Word[]{
                new Word("Hello", "Hello", null, null),
                new Word("guys", "guys", null, null),
        }, sentence.getWords());
        assertArrayEquals(new Tag[]{
                XMLTag.fromText("<a>", null, null, 1)
        }, sentence.getTags());
    }

    @Test
    public void testRequiredSpaceFalse() throws ProcessingException {
        String text = "Hello<a>!";
        Sentence sentence = process(text);

        assertEquals(text, sentence.toString(true, false));
        assertEquals("Hello!", sentence.toString(false, false));
        assertTrue(sentence.hasTags());

        assertArrayEquals(new Word[]{
                new Word("Hello", "Hello", null, null),
                new Word("!", "!", null, null),
        }, sentence.getWords());
        assertArrayEquals(new Tag[]{
                XMLTag.fromText("<a>", null, null, 1)
        }, sentence.getTags());
    }

    @Test
    public void testRequiredSpaceFalseWithRightSpace() throws ProcessingException {
        String text = "Hello<a> !";
        Sentence sentence = process(text);

        assertEquals(text, sentence.toString(true, false));
        assertEquals("Hello !", sentence.toString(false, false));
        assertTrue(sentence.hasTags());

        assertArrayEquals(new Word[]{
                new Word("Hello", "Hello", null, null),
                new Word("!", "!", " ", null),
        }, sentence.getWords());
        assertArrayEquals(new Tag[]{
                XMLTag.fromText("<a>", null, " ", 1)
        }, sentence.getTags());
    }

    @Test
    public void testRequiredSpaceFalseWithLeftSpace() throws ProcessingException {
        String text = "Hello <a>!";
        Sentence sentence = process(text);

        assertEquals(text, sentence.toString(true, false));
        assertEquals("Hello !", sentence.toString(false, false));
        assertTrue(sentence.hasTags());

        assertArrayEquals(new Word[]{
                new Word("Hello", "Hello", null, " "),
                new Word("!", "!", null, null),
        }, sentence.getWords());
        assertArrayEquals(new Tag[]{
                XMLTag.fromText("<a>", " ", null, 1)
        }, sentence.getTags());
    }

}
{'fraction_non_alphanumeric': 0.142411118944506, 'fraction_numerical': 0.0012085809245644074, 'mean_word_length': 2.5150442477876105, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5834493', 'n_tokens_mistral': 2708, 'n_tokens_neox': 2552, 'n_words': 600}