<commit_before>/**
 * Copyright (C) 2016 Hurence (<EMAIL>)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.hurence.logisland.processor;

import com.hurence.logisland.processor.util.BaseSyslogTest;
import com.hurence.logisland.record.FieldType;
import com.hurence.logisland.record.Record;
import com.hurence.logisland.record.StandardRecord;
import com.hurence.logisland.util.runner.MockRecord;
import com.hurence.logisland.util.runner.TestRunner;
import com.hurence.logisland.util.runner.TestRunners;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class RegexpProcessorTest extends BaseSyslogTest {

    private static final Logger logger = LoggerFactory.getLogger(RegexpProcessorTest.class);

    private Record getRecord1() {
        Record record1 = new StandardRecord();
        record1.setField("location", FieldType.STRING, "https://www.mycompany.com/fr/search");
        record1.setField("description", FieldType.STRING, "text1-text2");
        record1.setField("long1", FieldType.LONG, 1);
        record1.setField("long2", FieldType.LONG, 2);
        return record1;
    }

    @Test
    public void testNoRegexp() {
        Record record1 = getRecord1();

        TestRunner testRunner = TestRunners.newTestRunner(new RegexpProcessor());
        testRunner.setProperty("location", "companyName");
        testRunner.assertValid();
        testRunner.enqueue(record1);
        testRunner.run();
        testRunner.assertAllInputRecordsProcessed();
        testRunner.assertOutputRecordsCount(1);

        MockRecord out = testRunner.getOutputRecords().get(0);
        out.assertRecordSizeEquals(4);
    }

    @Test
    public void testEmptyRegexp() {
        Record record1 = getRecord1();

        TestRunner testRunner = TestRunners.newTestRunner(new RegexpProcessor());
        testRunner.setProperty("location", "companyName:");
        testRunner.assertValid();
        testRunner.enqueue(record1);
        testRunner.run();
        testRunner.assertAllInputRecordsProcessed();
        testRunner.assertOutputRecordsCount(1);

        MockRecord out = testRunner.getOutputRecords().get(0);
        out.assertRecordSizeEquals(4);
    }

    @Test
    public void testRegexpWithoutGroup() {
        Record record1 = getRecord1();

        TestRunner testRunner = TestRunners.newTestRunner(new RegexpProcessor());
        testRunner.setProperty("location", "companyName:nogroup");
        testRunner.assertValid();
        testRunner.enqueue(record1);
        testRunner.run();
        testRunner.assertAllInputRecordsProcessed();
        testRunner.assertOutputRecordsCount(1);

        MockRecord out = testRunner.getOutputRecords().get(0);
        out.assertRecordSizeEquals(4);
    }


    @Test
    public void testRegexpSimple() {

        Record record1 = getRecord1();

        TestRunner testRunner = TestRunners.newTestRunner(new RegexpProcessor());
        testRunner.setProperty("location", "companyName:https\\:\\/\\/www\\.(\\w+\\.\\w+)\\/?.*");
        testRunner.assertValid();
        testRunner.enqueue(record1);
        testRunner.run();
        testRunner.assertAllInputRecordsProcessed();
        testRunner.assertOutputRecordsCount(1);

        MockRecord out = testRunner.getOutputRecords().get(0);
        out.assertRecordSizeEquals(5);
        out.assertFieldEquals("companyName", "mycompany.com");
    }

    @Test
    public void testRegexpMultiple() {

        Record record1 = getRecord1();

        TestRunner testRunner = TestRunners.newTestRunner(new RegexpProcessor());
        testRunner.setProperty("location", "companyName:https\\:\\/\\/www\\.(\\w+\\.\\w+)\\/?.*");
        testRunner.setProperty("description", "part1,part2:(.+)-(.+)");
        testRunner.assertValid();
        testRunner.enqueue(record1);
        testRunner.run();
        testRunner.assertAllInputRecordsProcessed();
        testRunner.assertOutputRecordsCount(1);

        MockRecord out = testRunner.getOutputRecords().get(0);
        out.assertRecordSizeEquals(7);
        out.assertFieldEquals("companyName", "mycompany.com");
        out.assertFieldEquals("part1", "text1");
        out.assertFieldEquals("part2", "text2");
    }

    @Test
    public void testRegexpMultipleMoreOutputVarsThanGroups() {

        Record record1 = getRecord1();

        TestRunner testRunner = TestRunners.newTestRunner(new RegexpProcessor());
        testRunner.setProperty("location", "companyName:https\\:\\/\\/www\\.(\\w+\\.\\w+)\\/?.*");
        testRunner.setProperty("description", "part1,part2,part3:(.+)-(.+)");
        testRunner.assertValid();
        testRunner.enqueue(record1);
        testRunner.run();
        testRunner.assertAllInputRecordsProcessed();
        testRunner.assertOutputRecordsCount(1);

        MockRecord out = testRunner.getOutputRecords().get(0);
        out.assertRecordSizeEquals(7);
        out.assertFieldEquals("companyName", "mycompany.com");
        out.assertFieldEquals("part1", "text1");
        out.assertFieldEquals("part2", "text2");
    }

    @Test
    public void testRegexpMultipleLessOutputVarsThanGroups() {

        Record record1 = getRecord1();

        TestRunner testRunner = TestRunners.newTestRunner(new RegexpProcessor());
        testRunner.setProperty("location", "companyName:https\\:\\/\\/www\\.(\\w+\\.\\w+)\\/?.*");
        testRunner.setProperty("description", "part1:(.+)-(.+)");
        testRunner.assertValid();
        testRunner.enqueue(record1);
        testRunner.run();<commit_msg>Minor changes: - Rename the getRecord1() method in getRecord() - Remove use of BaseSyslogTest in the Unit Test of RegexpProcessor<commit_after>/**
 * Copyright (C) 2016 Hurence (<EMAIL>)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.hurence.logisland.processor;

import com.hurence.logisland.record.FieldType;
import com.hurence.logisland.record.Record;
import com.hurence.logisland.record.StandardRecord;
import com.hurence.logisland.util.runner.MockRecord;
import com.hurence.logisland.util.runner.TestRunner;
import com.hurence.logisland.util.runner.TestRunners;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class RegexpProcessorTest {

    private static final Logger logger = LoggerFactory.getLogger(RegexpProcessorTest.class);

    private Record getRecord() {
        Record record1 = new StandardRecord();
        record1.setField("location", FieldType.STRING, "https://www.mycompany.com/fr/search");
        record1.setField("description", FieldType.STRING, "text1-text2");
        record1.setField("long1", FieldType.LONG, 1);
        record1.setField("long2", FieldType.LONG, 2);
        return record1;
    }

    @Test
    public void testNoRegexp() {
        Record record1 = getRecord();

        TestRunner testRunner = TestRunners.newTestRunner(new RegexpProcessor());
        testRunner.setProperty("location", "companyName");
        testRunner.assertValid();
        testRunner.enqueue(record1);
        testRunner.run();
        testRunner.assertAllInputRecordsProcessed();
        testRunner.assertOutputRecordsCount(1);

        MockRecord out = testRunner.getOutputRecords().get(0);
        out.assertRecordSizeEquals(4);
    }

    @Test
    public void testEmptyRegexp() {
        Record record1 = getRecord();

        TestRunner testRunner = TestRunners.newTestRunner(new RegexpProcessor());
        testRunner.setProperty("location", "companyName:");
        testRunner.assertValid();
        testRunner.enqueue(record1);
        testRunner.run();
        testRunner.assertAllInputRecordsProcessed();
        testRunner.assertOutputRecordsCount(1);

        MockRecord out = testRunner.getOutputRecords().get(0);
        out.assertRecordSizeEquals(4);
    }

    @Test
    public void testRegexpWithoutGroup() {
        Record record1 = getRecord();

        TestRunner testRunner = TestRunners.newTestRunner(new RegexpProcessor());
        testRunner.setProperty("location", "companyName:nogroup");
        testRunner.assertValid();
        testRunner.enqueue(record1);
        testRunner.run();
        testRunner.assertAllInputRecordsProcessed();
        testRunner.assertOutputRecordsCount(1);

        MockRecord out = testRunner.getOutputRecords().get(0);
        out.assertRecordSizeEquals(4);
    }


    @Test
    public void testRegexpSimple() {

        Record record1 = getRecord();

        TestRunner testRunner = TestRunners.newTestRunner(new RegexpProcessor());
        testRunner.setProperty("location", "companyName:https\\:\\/\\/www\\.(\\w+\\.\\w+)\\/?.*");
        testRunner.assertValid();
        testRunner.enqueue(record1);
        testRunner.run();
        testRunner.assertAllInputRecordsProcessed();
        testRunner.assertOutputRecordsCount(1);

        MockRecord out = testRunner.getOutputRecords().get(0);
        out.assertRecordSizeEquals(5);
        out.assertFieldEquals("companyName", "mycompany.com");
    }

    @Test
    public void testRegexpMultiple() {

        Record record1 = getRecord();

        TestRunner testRunner = TestRunners.newTestRunner(new RegexpProcessor());
        testRunner.setProperty("location", "companyName:https\\:\\/\\/www\\.(\\w+\\.\\w+)\\/?.*");
        testRunner.setProperty("description", "part1,part2:(.+)-(.+)");
        testRunner.assertValid();
        testRunner.enqueue(record1);
        testRunner.run();
        testRunner.assertAllInputRecordsProcessed();
        testRunner.assertOutputRecordsCount(1);

        MockRecord out = testRunner.getOutputRecords().get(0);
        out.assertRecordSizeEquals(7);
        out.assertFieldEquals("companyName", "mycompany.com");
        out.assertFieldEquals("part1", "text1");
        out.assertFieldEquals("part2", "text2");
    }

    @Test
    public void testRegexpMultipleMoreOutputVarsThanGroups() {

        Record record1 = getRecord();

        TestRunner testRunner = TestRunners.newTestRunner(new RegexpProcessor());
        testRunner.setProperty("location", "companyName:https\\:\\/\\/www\\.(\\w+\\.\\w+)\\/?.*");
        testRunner.setProperty("description", "part1,part2,part3:(.+)-(.+)");
        testRunner.assertValid();
        testRunner.enqueue(record1);
        testRunner.run();
        testRunner.assertAllInputRecordsProcessed();
        testRunner.assertOutputRecordsCount(1);

        MockRecord out = testRunner.getOutputRecords().get(0);
        out.assertRecordSizeEquals(7);
        out.assertFieldEquals("companyName", "mycompany.com");
        out.assertFieldEquals("part1", "text1");
        out.assertFieldEquals("part2", "text2");
    }

    @Test
    public void testRegexpMultipleLessOutputVarsThanGroups() {

        Record record1 = getRecord();

        TestRunner testRunner = TestRunners.newTestRunner(new RegexpProcessor());
        testRunner.setProperty("location", "companyName:https\\:\\/\\/www\\.(\\w+\\.\\w+)\\/?.*");
        testRunner.setProperty("description", "part1:(.+)-(.+)");
        testRunner.assertValid();
        testRunner.enqueue(record1);
        testRunner.run();
{'fraction_non_alphanumeric': 0.13121124431583298, 'fraction_numerical': 0.011988424968995453, 'mean_word_length': 3.290883291947499, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 12, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4160460', 'n_tokens_mistral': 3478, 'n_tokens_neox': 3370, 'n_words': 624}