<commit_before>/*
 *  * Copyright (C) 2018 Hurence (<EMAIL>)
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package com.hurence.logisland.connect.converter;

import com.hurence.logisland.record.Field;
import com.hurence.logisland.record.FieldDictionary;
import com.hurence.logisland.record.Record;
import com.hurence.logisland.serializer.BytesArraySerializer;
import com.hurence.logisland.serializer.KryoSerializer;
import com.hurence.logisland.serializer.RecordSerializer;
import com.hurence.logisland.serializer.SerializerProvider;
import org.apache.kafka.connect.data.Schema;
import org.apache.kafka.connect.data.SchemaBuilder;
import org.apache.kafka.connect.data.Struct;
import org.junit.Test;

import java.io.ByteArrayInputStream;
import java.util.*;

import static org.junit.Assert.*;

public class LogIslandRecordConverterTest {

    private LogIslandRecordConverter setupInstance(Class<? extends RecordSerializer> serializerClass, boolean isKey) {
        final LogIslandRecordConverter instance = new LogIslandRecordConverter();
        instance.configure(
                Collections.singletonMap(LogIslandRecordConverter.PROPERTY_RECORD_SERIALIZER, serializerClass.getCanonicalName()),
                isKey);
        return instance;
    }

    private void assertFieldEquals(Record record, String fieldName, Object expected) {
        Field field = record.getField(fieldName);
        assertNotNull(field);
        assertEquals(expected, record.getField(fieldName).getRawValue());
    }

    private void assertFieldEquals(Record record, String fieldName, byte[] expected) {
        Field field = record.getField(fieldName);
        assertNotNull(field);
        assertArrayEquals(expected, (byte[]) record.getField(fieldName).getRawValue());
    }
<commit_msg>Fix unit testing<commit_after>/*
 *  * Copyright (C) 2018 Hurence (<EMAIL>)
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package com.hurence.logisland.connect.converter;

import com.hurence.logisland.record.Field;
import com.hurence.logisland.record.FieldDictionary;
import com.hurence.logisland.record.Record;
import com.hurence.logisland.serializer.BytesArraySerializer;
import com.hurence.logisland.serializer.KryoSerializer;
import com.hurence.logisland.serializer.RecordSerializer;
import com.hurence.logisland.serializer.SerializerProvider;
import org.apache.kafka.connect.data.Schema;
import org.apache.kafka.connect.data.SchemaBuilder;
import org.apache.kafka.connect.data.Struct;
import org.junit.Test;

import java.io.ByteArrayInputStream;
import java.util.*;

import static org.junit.Assert.*;

public class LogIslandRecordConverterTest {

    private LogIslandRecordConverter setupInstance(Class<? extends RecordSerializer> serializerClass, boolean isKey) {
        final LogIslandRecordConverter instance = new LogIslandRecordConverter();
        instance.configure(
                Collections.singletonMap(LogIslandRecordConverter.PROPERTY_RECORD_SERIALIZER, serializerClass.getCanonicalName()),
                isKey);
        return instance;
    }

    private void assertFieldEquals(Record record, String fieldName, Object expected) {
        Field field = record.getField(fieldName);
        if (expected == null) {
            assertNull(field);
        } else {
            assertNotNull(field);
            assertEquals(expected, field.getRawValue());
        }
    }

    private void assertFieldEquals(Record record, String fieldName, byte[] expected) {
        Field field = record.getField(fieldName);
        if (expected == null) {
            assertNull(field);
        } else {
            assertNotNull(field);
            assertArrayEquals(expected, (byte[]) field.getRawValue());
        }
    }

{'fraction_non_alphanumeric': 0.08985933235355868, 'fraction_numerical': 0.003359227377703128, 'mean_word_length': 3.846388606307223, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4946602', 'n_tokens_mistral': 1305, 'n_tokens_neox': 1268, 'n_words': 344}