<commit_before>package org.myrobotlab.service.interfaces;

import org.junit.Assert;
import org.junit.Test;
import org.myrobotlab.document.connector.AbstractConnector;
import org.myrobotlab.document.connector.ConnectorState;
import org.myrobotlab.framework.Service;
import org.myrobotlab.logging.LoggerFactory;
import org.slf4j.Logger;

public abstract class AbstractConnectorTest {

	public final static Logger log = LoggerFactory.getLogger(AbstractConnectorTest.class);
	public abstract AbstractConnector createConnector();
	public abstract MockDocumentListener createListener();
	public abstract void validate(MockDocumentListener listener);
	
	@Test
	public void test() {
		AbstractConnector connector = createConnector();
		MockDocumentListener listener = createListener();
		connector.startService();
		listener.startService();
		connector.addDocumentListener(listener);
		connector.startCrawling();
		// flush any current batch and wait until the outbox is empty.
		connector.flush();
		System.out.println("Done Crawling");
		
		// Wait for the inbox to drain<commit_msg>ignroing the abstract base test class for connectors<commit_after>package org.myrobotlab.service.interfaces;

import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
import org.myrobotlab.document.connector.AbstractConnector;
import org.myrobotlab.document.connector.ConnectorState;
import org.myrobotlab.framework.Service;
import org.myrobotlab.logging.LoggerFactory;
import org.slf4j.Logger;

@Ignore
public abstract class AbstractConnectorTest {

	public final static Logger log = LoggerFactory.getLogger(AbstractConnectorTest.class);
	public abstract AbstractConnector createConnector();
	public abstract MockDocumentListener createListener();
	public abstract void validate(MockDocumentListener listener);
	
	@Test
	public void test() {
		AbstractConnector connector = createConnector();
		MockDocumentListener listener = createListener();
		connector.startService();
		listener.startService();
		connector.addDocumentListener(listener);
		connector.startCrawling();
		// flush any current batch and wait until the outbox is empty.
		connector.flush();
		System.out.println("Done Crawling");
		
		// Wait for the inbox to drain
{'fraction_non_alphanumeric': 0.0862533692722372, 'fraction_numerical': 0.0008984725965858042, 'mean_word_length': 6.115015974440895, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '937738', 'n_tokens_mistral': 606, 'n_tokens_neox': 584, 'n_words': 155}