<commit_before>package com.thinkbiganalytics.integration.feed;

/*-
 * #%L
 * kylo-feed-manager-controller
 * %%
 * Copyright (C) 2017 ThinkBig Analytics
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *     http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * #L%
 */

import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.module.SimpleModule;
import com.google.common.base.CharMatcher;
import com.thinkbiganalytics.discovery.model.DefaultHiveSchema;
import com.thinkbiganalytics.discovery.model.DefaultTableSchema;
import com.thinkbiganalytics.discovery.model.DefaultTag;
import com.thinkbiganalytics.discovery.schema.Field;
import com.thinkbiganalytics.discovery.schema.Tag;
import com.thinkbiganalytics.feedmgr.rest.model.FeedCategory;
import com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata;
import com.thinkbiganalytics.feedmgr.rest.model.FeedSchedule;
import com.thinkbiganalytics.feedmgr.rest.model.RegisteredTemplate;
import com.thinkbiganalytics.feedmgr.rest.model.schema.FeedProcessingOptions;
import com.thinkbiganalytics.feedmgr.rest.model.schema.PartitionField;
import com.thinkbiganalytics.feedmgr.rest.model.schema.TableOptions;
import com.thinkbiganalytics.feedmgr.rest.model.schema.TableSetup;
import com.thinkbiganalytics.feedmgr.service.feed.ExportImportFeedService;
import com.thinkbiganalytics.feedmgr.service.template.ExportImportTemplateService;
import com.thinkbiganalytics.integration.IntegrationTestBase;
import com.thinkbiganalytics.jobrepo.query.model.DefaultExecutedJob;
import com.thinkbiganalytics.jobrepo.query.model.DefaultExecutedStep;
import com.thinkbiganalytics.jobrepo.query.model.ExecutedStep;
import com.thinkbiganalytics.jobrepo.query.model.ExecutionStatus;
import com.thinkbiganalytics.jobrepo.query.model.ExitStatus;
import com.thinkbiganalytics.nifi.rest.model.NifiProperty;
import com.thinkbiganalytics.policy.rest.model.FieldPolicy;
import com.thinkbiganalytics.policy.rest.model.FieldStandardizationRule;
import com.thinkbiganalytics.policy.rest.model.FieldValidationRule;
import com.thinkbiganalytics.security.rest.model.User;

import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;

/**
 * Basic Feed Integration Test which imports data index feed, creates category, imports data ingest template,
 * creates data ingest feed, runs the feed, validates number of executed jobs, validates validators and
 * standardisers have been applied by looking at profiler summary, validates total number and number of
 * valid and invalid rows, validates expected hive tables have been created and runs a simple hive
 * query and asserts the number of rows returned.
 *
 */
public class FeedIT extends IntegrationTestBase {

    private static final Logger LOG = LoggerFactory.getLogger(FeedIT.class);

    private static final String SAMPLES_DIR = "/samples";
    private static final String DATA_SAMPLES_DIR = SAMPLES_DIR + "/sample-data/csv/";
    private static final String NIFI_VERSION = "nifi-1.0";
    private static final String TEMPLATE_SAMPLES_DIR = SAMPLES_DIR + "/templates/" + NIFI_VERSION + "/";
    private static final String FEED_SAMPLES_DIR = SAMPLES_DIR + "/feeds/" + NIFI_VERSION + "/";
    protected static final String DATA_INGEST_ZIP = "data_ingest.zip";
    private static final String VAR_DROPZONE = "/var/dropzone";
    private static final String USERDATA1_CSV = "userdata1.csv";
    private static final int FEED_COMPLETION_WAIT_DELAY = 180;
    private static final int VALID_RESULTS = 879;
    private static String FEED_NAME = "users_" + System.currentTimeMillis();

    private String sampleFeedsPath;
    protected String sampleTemplatesPath;
    private String usersDataPath;

    private FieldStandardizationRule toUpperCase = new FieldStandardizationRule();
    private FieldValidationRule email = new FieldValidationRule();
    private FieldValidationRule lookup = new FieldValidationRule();
    private FieldValidationRule notNull = new FieldValidationRule();
    private FieldStandardizationRule base64EncodeBinary = new FieldStandardizationRule();
    private FieldStandardizationRule base64EncodeString = new FieldStandardizationRule();
    private FieldStandardizationRule base64DecodeBinary = new FieldStandardizationRule();
    private FieldStandardizationRule base64DecodeString = new FieldStandardizationRule();
    private FieldValidationRule length = new FieldValidationRule();
    private FieldValidationRule ipAddress = new FieldValidationRule();

    @Override
    protected void configureObjectMapper(ObjectMapper om) {
        SimpleModule m = new SimpleModule();
        m.addAbstractTypeMapping(ExecutedStep.class, DefaultExecutedStep.class);
        om.registerModule(m);
    }

    @Test
    public void testDataIngestFeed() throws Exception {
        prepare();

        importSystemFeeds();

        copyDataToDropzone();

        //create new category
        FeedCategory category = createCategory("Functional Tests");

        ExportImportTemplateService.ImportTemplate ingest = importDataIngestTemplate();

        //create standard ingest feed
        FeedMetadata feed = getCreateFeedRequest(category, ingest, FEED_NAME);
        FeedMetadata response = createFeed(feed).getFeedMetadata();
        Assert.assertEquals(feed.getFeedName(), response.getFeedName());

        waitForFeedToComplete();

        assertExecutedJobs(response);

        failJobs(response.getCategoryAndFeedName());
        abandonAllJobs(response.getCategoryAndFeedName());
    }

    @Override
    public void startClean() {
        super.startClean();
    }

//    @Test
    public void temp() {
//        failJobs("system.index_text_service");
    }

    protected void prepare() throws Exception {
        String path = getClass().getResource(".").toURI().getPath();
        String basedir = path.substring(0, path.indexOf("services"));
        sampleFeedsPath = basedir + FEED_SAMPLES_DIR;
        sampleTemplatesPath = basedir + TEMPLATE_SAMPLES_DIR;
        usersDataPath = basedir + DATA_SAMPLES_DIR;

        toUpperCase.setName("Uppercase");
        toUpperCase.setDisplayName("Uppercase");
        toUpperCase.setDescription("Convert string to uppercase");
        toUpperCase.setObjectClassType("com.thinkbiganalytics.policy.standardization.UppercaseStandardizer");
        toUpperCase.setObjectShortClassType("UppercaseStandardizer");

        email.setName("email");
        email.setDisplayName("Email");
        email.setDescription("Valid email address");
        email.setObjectClassType("com.thinkbiganalytics.policy.validation.EmailValidator");
        email.setObjectShortClassType("EmailValidator");

        ipAddress.setName("IP Address");
        ipAddress.setDisplayName("IP Address");
        ipAddress.setDescription("Valid IP address");
        ipAddress.setObjectClassType("com.thinkbiganalytics.policy.validation.IPAddressValidator");
        ipAddress.setObjectShortClassType("IPAddressValidator");

        lookup.setName("lookup");
        lookup.setDisplayName("Lookup");
        lookup.setDescription("Must be contained in the list");
        lookup.setObjectClassType("com.thinkbiganalytics.policy.validation.LookupValidator");
        lookup.setObjectShortClassType("LookupValidator");
        lookup.setProperties(newFieldRuleProperties(newFieldRuleProperty("List", "lookupList", "Male,Female")));

        base64DecodeBinary.setName("Base64 Decode");
        base64DecodeBinary.setDisplayName("Base64 Decode");
        base64DecodeBinary.setDescription("Base64 decode a string or a byte[].  Strings are evaluated using the UTF-8 charset");
        base64DecodeBinary.setObjectClassType("com.thinkbiganalytics.policy.standardization.Base64Decode");
        base64DecodeBinary.setObjectShortClassType("Base64Decode");
        base64DecodeBinary.setProperties(newFieldRuleProperties(newFieldRuleProperty("Output", "base64Output", "BINARY")));

        base64DecodeString.setName("Base64 Decode");
        base64DecodeString.setDisplayName("Base64 Decode");
        base64DecodeString.setDescription("Base64 decode a string or a byte[].  Strings are evaluated using the UTF-8 charset");
        base64DecodeString.setObjectClassType("com.thinkbiganalytics.policy.standardization.Base64Decode");
        base64DecodeString.setObjectShortClassType("Base64Decode");
        base64DecodeString.setProperties(newFieldRuleProperties(newFieldRuleProperty("Output", "base64Output", "STRING")));

        base64EncodeBinary.setName("Base64 Encode");
        base64EncodeBinary.setDisplayName("Base64 Encode");
        base64EncodeBinary.setDescription("Base64 encode a string or a byte[].  Strings are evaluated using the UTF-8 charset.  String output is urlsafe");
        base64EncodeBinary.setObjectClassType("com.thinkbiganalytics.policy.standardization.Base64Encode");
        base64EncodeBinary.setObjectShortClassType("Base64Encode");
        base64EncodeBinary.setProperties(newFieldRuleProperties(newFieldRuleProperty("Output", "base64Output", "BINARY")));

        base64EncodeString.setName("Base64 Encode");
        base64EncodeString.setDisplayName("Base64 Encode");
        base64EncodeString.setDescription("Base64 encode a string or a byte[].  Strings are evaluated using the UTF-8 charset.  String output is urlsafe");
        base64EncodeString.setObjectClassType("com.thinkbiganalytics.policy.standardization.Base64Encode");
        base64EncodeString.setObjectShortClassType("Base64Encode");
        base64EncodeString.setProperties(newFieldRuleProperties(newFieldRuleProperty("Output", "base64Output", "STRING")));

        notNull.setName("Not Null");
        notNull.setDisplayName("Not Null");
        notNull.setDescription("Validate a value is not null");
        notNull.setObjectClassType("com.thinkbiganalytics.policy.validation.NotNullValidator");
        notNull.setObjectShortClassType("NotNullValidator");
        notNull.setProperties(newFieldRuleProperties(newFieldRuleProperty("EMPTY_STRING", "allowEmptyString", "false"),
                                                     newFieldRuleProperty("TRIM_STRING", "trimString", "true")));

        length.setName("Length");
        length.setDisplayName("Length");
        length.setDescription("Validate String falls between desired length");
        length.setObjectClassType("com.thinkbiganalytics.policy.validation.LengthValidator");
        length.setObjectShortClassType("LengthValidator");
        length.setProperties(newFieldRuleProperties(newFieldRuleProperty("Max Length", "maxLength", "15"),
                                                    newFieldRuleProperty("Min Length", "minLength", "5")));
    }


    protected void importSystemFeeds() {
        ExportImportFeedService.ImportFeed textIndex = importFeed(sampleFeedsPath + "index_text_service_elasticsearch.feed.zip");
        enableFeed(textIndex.getNifiFeed().getFeedMetadata().getFeedId());
    }

    protected ExportImportTemplateService.ImportTemplate importDataIngestTemplate() {
        return importFeedTemplate(sampleTemplatesPath + DATA_INGEST_ZIP);
    }

    protected ExportImportTemplateService.ImportTemplate importFeedTemplate(String templatePath) {
        LOG.info("Importing feed template {}", templatePath);

        //get number of templates already there
        int existingTemplateNum = getTemplates().length;

        //import standard feedTemplate template
        ExportImportTemplateService.ImportTemplate feedTemplate = importTemplate(templatePath);
        Assert.assertTrue(templatePath.contains(feedTemplate.getFileName()));
        Assert.assertTrue(feedTemplate.isSuccess());

        //assert new template is there
        RegisteredTemplate[] templates = getTemplates();
        Assert.assertTrue(templates.length == existingTemplateNum + 1);
        return feedTemplate;
    }

    protected void copyDataToDropzone() {
        LOG.info("Copying data to dropzone");<commit_msg>Fix Elastic Search version and Text Index Feed version mismatch in Integration tests<commit_after>package com.thinkbiganalytics.integration.feed;

/*-
 * #%L
 * kylo-feed-manager-controller
 * %%
 * Copyright (C) 2017 ThinkBig Analytics
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *     http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * #L%
 */

import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.module.SimpleModule;
import com.google.common.base.CharMatcher;
import com.thinkbiganalytics.discovery.model.DefaultHiveSchema;
import com.thinkbiganalytics.discovery.model.DefaultTableSchema;
import com.thinkbiganalytics.discovery.model.DefaultTag;
import com.thinkbiganalytics.discovery.schema.Field;
import com.thinkbiganalytics.discovery.schema.Tag;
import com.thinkbiganalytics.feedmgr.rest.model.FeedCategory;
import com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata;
import com.thinkbiganalytics.feedmgr.rest.model.FeedSchedule;
import com.thinkbiganalytics.feedmgr.rest.model.RegisteredTemplate;
import com.thinkbiganalytics.feedmgr.rest.model.schema.FeedProcessingOptions;
import com.thinkbiganalytics.feedmgr.rest.model.schema.PartitionField;
import com.thinkbiganalytics.feedmgr.rest.model.schema.TableOptions;
import com.thinkbiganalytics.feedmgr.rest.model.schema.TableSetup;
import com.thinkbiganalytics.feedmgr.service.feed.ExportImportFeedService;
import com.thinkbiganalytics.feedmgr.service.template.ExportImportTemplateService;
import com.thinkbiganalytics.integration.IntegrationTestBase;
import com.thinkbiganalytics.jobrepo.query.model.DefaultExecutedJob;
import com.thinkbiganalytics.jobrepo.query.model.DefaultExecutedStep;
import com.thinkbiganalytics.jobrepo.query.model.ExecutedStep;
import com.thinkbiganalytics.jobrepo.query.model.ExecutionStatus;
import com.thinkbiganalytics.jobrepo.query.model.ExitStatus;
import com.thinkbiganalytics.nifi.rest.model.NifiProperty;
import com.thinkbiganalytics.policy.rest.model.FieldPolicy;
import com.thinkbiganalytics.policy.rest.model.FieldStandardizationRule;
import com.thinkbiganalytics.policy.rest.model.FieldValidationRule;
import com.thinkbiganalytics.security.rest.model.User;

import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;

/**
 * Basic Feed Integration Test which imports data index feed, creates category, imports data ingest template,
 * creates data ingest feed, runs the feed, validates number of executed jobs, validates validators and
 * standardisers have been applied by looking at profiler summary, validates total number and number of
 * valid and invalid rows, validates expected hive tables have been created and runs a simple hive
 * query and asserts the number of rows returned.
 *
 */
public class FeedIT extends IntegrationTestBase {

    private static final Logger LOG = LoggerFactory.getLogger(FeedIT.class);

    private static final String SAMPLES_DIR = "/samples";
    private static final String DATA_SAMPLES_DIR = SAMPLES_DIR + "/sample-data/csv/";
    private static final String NIFI_FEED_SAMPLE_VERSION = "nifi-1.3";
    private static final String NIFI_TEMPLATE_SAMPLE_VERSION = "nifi-1.0";
    private static final String TEMPLATE_SAMPLES_DIR = SAMPLES_DIR + "/templates/" + NIFI_TEMPLATE_SAMPLE_VERSION + "/";
    private static final String FEED_SAMPLES_DIR = SAMPLES_DIR + "/feeds/" + NIFI_FEED_SAMPLE_VERSION + "/";
    protected static final String DATA_INGEST_ZIP = "data_ingest.zip";
    private static final String VAR_DROPZONE = "/var/dropzone";
    private static final String USERDATA1_CSV = "userdata1.csv";
    private static final int FEED_COMPLETION_WAIT_DELAY = 180;
    private static final int VALID_RESULTS = 879;
    private static final String INDEX_TEXT_SERVICE_V2_FEED_ZIP = "index_text_service_v2.feed.zip";
    private static String FEED_NAME = "users_" + System.currentTimeMillis();

    private String sampleFeedsPath;
    protected String sampleTemplatesPath;
    private String usersDataPath;

    private FieldStandardizationRule toUpperCase = new FieldStandardizationRule();
    private FieldValidationRule email = new FieldValidationRule();
    private FieldValidationRule lookup = new FieldValidationRule();
    private FieldValidationRule notNull = new FieldValidationRule();
    private FieldStandardizationRule base64EncodeBinary = new FieldStandardizationRule();
    private FieldStandardizationRule base64EncodeString = new FieldStandardizationRule();
    private FieldStandardizationRule base64DecodeBinary = new FieldStandardizationRule();
    private FieldStandardizationRule base64DecodeString = new FieldStandardizationRule();
    private FieldValidationRule length = new FieldValidationRule();
    private FieldValidationRule ipAddress = new FieldValidationRule();

    @Override
    protected void configureObjectMapper(ObjectMapper om) {
        SimpleModule m = new SimpleModule();
        m.addAbstractTypeMapping(ExecutedStep.class, DefaultExecutedStep.class);
        om.registerModule(m);
    }

    @Test
    public void testDataIngestFeed() throws Exception {
        prepare();

        importSystemFeeds();

        copyDataToDropzone();

        //create new category
        FeedCategory category = createCategory("Functional Tests");

        ExportImportTemplateService.ImportTemplate ingest = importDataIngestTemplate();

        //create standard ingest feed
        FeedMetadata feed = getCreateFeedRequest(category, ingest, FEED_NAME);
        FeedMetadata response = createFeed(feed).getFeedMetadata();
        Assert.assertEquals(feed.getFeedName(), response.getFeedName());

        waitForFeedToComplete();

        assertExecutedJobs(response);

        failJobs(response.getCategoryAndFeedName());
        abandonAllJobs(response.getCategoryAndFeedName());
    }

    @Override
    public void startClean() {
        super.startClean();
    }

//    @Test
    public void temp() {
//        failJobs("system.index_text_service");
    }

    protected void prepare() throws Exception {
        String path = getClass().getResource(".").toURI().getPath();
        String basedir = path.substring(0, path.indexOf("services"));
        sampleFeedsPath = basedir + FEED_SAMPLES_DIR;
        sampleTemplatesPath = basedir + TEMPLATE_SAMPLES_DIR;
        usersDataPath = basedir + DATA_SAMPLES_DIR;

        toUpperCase.setName("Uppercase");
        toUpperCase.setDisplayName("Uppercase");
        toUpperCase.setDescription("Convert string to uppercase");
        toUpperCase.setObjectClassType("com.thinkbiganalytics.policy.standardization.UppercaseStandardizer");
        toUpperCase.setObjectShortClassType("UppercaseStandardizer");

        email.setName("email");
        email.setDisplayName("Email");
        email.setDescription("Valid email address");
        email.setObjectClassType("com.thinkbiganalytics.policy.validation.EmailValidator");
        email.setObjectShortClassType("EmailValidator");

        ipAddress.setName("IP Address");
        ipAddress.setDisplayName("IP Address");
        ipAddress.setDescription("Valid IP address");
        ipAddress.setObjectClassType("com.thinkbiganalytics.policy.validation.IPAddressValidator");
        ipAddress.setObjectShortClassType("IPAddressValidator");

        lookup.setName("lookup");
        lookup.setDisplayName("Lookup");
        lookup.setDescription("Must be contained in the list");
        lookup.setObjectClassType("com.thinkbiganalytics.policy.validation.LookupValidator");
        lookup.setObjectShortClassType("LookupValidator");
        lookup.setProperties(newFieldRuleProperties(newFieldRuleProperty("List", "lookupList", "Male,Female")));

        base64DecodeBinary.setName("Base64 Decode");
        base64DecodeBinary.setDisplayName("Base64 Decode");
        base64DecodeBinary.setDescription("Base64 decode a string or a byte[].  Strings are evaluated using the UTF-8 charset");
        base64DecodeBinary.setObjectClassType("com.thinkbiganalytics.policy.standardization.Base64Decode");
        base64DecodeBinary.setObjectShortClassType("Base64Decode");
        base64DecodeBinary.setProperties(newFieldRuleProperties(newFieldRuleProperty("Output", "base64Output", "BINARY")));

        base64DecodeString.setName("Base64 Decode");
        base64DecodeString.setDisplayName("Base64 Decode");
        base64DecodeString.setDescription("Base64 decode a string or a byte[].  Strings are evaluated using the UTF-8 charset");
        base64DecodeString.setObjectClassType("com.thinkbiganalytics.policy.standardization.Base64Decode");
        base64DecodeString.setObjectShortClassType("Base64Decode");
        base64DecodeString.setProperties(newFieldRuleProperties(newFieldRuleProperty("Output", "base64Output", "STRING")));

        base64EncodeBinary.setName("Base64 Encode");
        base64EncodeBinary.setDisplayName("Base64 Encode");
        base64EncodeBinary.setDescription("Base64 encode a string or a byte[].  Strings are evaluated using the UTF-8 charset.  String output is urlsafe");
        base64EncodeBinary.setObjectClassType("com.thinkbiganalytics.policy.standardization.Base64Encode");
        base64EncodeBinary.setObjectShortClassType("Base64Encode");
        base64EncodeBinary.setProperties(newFieldRuleProperties(newFieldRuleProperty("Output", "base64Output", "BINARY")));

        base64EncodeString.setName("Base64 Encode");
        base64EncodeString.setDisplayName("Base64 Encode");
        base64EncodeString.setDescription("Base64 encode a string or a byte[].  Strings are evaluated using the UTF-8 charset.  String output is urlsafe");
        base64EncodeString.setObjectClassType("com.thinkbiganalytics.policy.standardization.Base64Encode");
        base64EncodeString.setObjectShortClassType("Base64Encode");
        base64EncodeString.setProperties(newFieldRuleProperties(newFieldRuleProperty("Output", "base64Output", "STRING")));

        notNull.setName("Not Null");
        notNull.setDisplayName("Not Null");
        notNull.setDescription("Validate a value is not null");
        notNull.setObjectClassType("com.thinkbiganalytics.policy.validation.NotNullValidator");
        notNull.setObjectShortClassType("NotNullValidator");
        notNull.setProperties(newFieldRuleProperties(newFieldRuleProperty("EMPTY_STRING", "allowEmptyString", "false"),
                                                     newFieldRuleProperty("TRIM_STRING", "trimString", "true")));

        length.setName("Length");
        length.setDisplayName("Length");
        length.setDescription("Validate String falls between desired length");
        length.setObjectClassType("com.thinkbiganalytics.policy.validation.LengthValidator");
        length.setObjectShortClassType("LengthValidator");
        length.setProperties(newFieldRuleProperties(newFieldRuleProperty("Max Length", "maxLength", "15"),
                                                    newFieldRuleProperty("Min Length", "minLength", "5")));
    }


    protected void importSystemFeeds() {
        ExportImportFeedService.ImportFeed textIndex = importFeed(sampleFeedsPath + INDEX_TEXT_SERVICE_V2_FEED_ZIP);
        enableFeed(textIndex.getNifiFeed().getFeedMetadata().getFeedId());
    }

    protected ExportImportTemplateService.ImportTemplate importDataIngestTemplate() {
        return importFeedTemplate(sampleTemplatesPath + DATA_INGEST_ZIP);
    }

    protected ExportImportTemplateService.ImportTemplate importFeedTemplate(String templatePath) {
        LOG.info("Importing feed template {}", templatePath);

        //get number of templates already there
        int existingTemplateNum = getTemplates().length;

        //import standard feedTemplate template
        ExportImportTemplateService.ImportTemplate feedTemplate = importTemplate(templatePath);
        Assert.assertTrue(templatePath.contains(feedTemplate.getFileName()));
        Assert.assertTrue(feedTemplate.isSuccess());

        //assert new template is there
        RegisteredTemplate[] templates = getTemplates();
        Assert.assertTrue(templates.length == existingTemplateNum + 1);
        return feedTemplate;
    }

    protected void copyDataToDropzone() {
        LOG.info("Copying data to dropzone");
{'fraction_non_alphanumeric': 0.09171296658580484, 'fraction_numerical': 0.010603333594177948, 'mean_word_length': 4.523881564728766, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1120373', 'n_tokens_mistral': 6819, 'n_tokens_neox': 6599, 'n_words': 1409}