<commit_before>package eu.socialsensor.sfc.streams.management;


import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;

import javax.xml.parsers.ParserConfigurationException;

import org.apache.log4j.Logger;
import org.xml.sax.SAXException;

import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig;
import redis.clients.jedis.JedisPubSub;

import eu.socialsensor.framework.client.search.solr.SolrDyscoHandler;
import eu.socialsensor.framework.common.domain.Feed;
import eu.socialsensor.framework.common.domain.dysco.Dysco;
import eu.socialsensor.framework.common.domain.dysco.Dysco.DyscoType;
import eu.socialsensor.framework.common.domain.dysco.Message;
import eu.socialsensor.framework.common.domain.dysco.Message.Action;
import eu.socialsensor.framework.streams.Stream;
import eu.socialsensor.framework.streams.StreamConfiguration;
import eu.socialsensor.framework.streams.StreamException;
import eu.socialsensor.sfc.builder.FeedsCreator;
import eu.socialsensor.sfc.builder.input.DataInputType;
import eu.socialsensor.sfc.streams.StreamsManagerConfiguration;
import eu.socialsensor.sfc.streams.monitors.StreamsMonitor;

public class MediaSearcher {
	private static String REDIS_HOST = "redis.host";
	private static String SOLR_HOST = "solr.hostname";
	private static String SOLR_SERVICE = "solr.service";
	private static String DYSCO_COLLECTION = "dyscos.collection";
	
	
	
	public final Logger logger = Logger.getLogger(StreamsManager.class);
	
	enum MediaSearcherState {
		OPEN, CLOSE
	}
	private MediaSearcherState state = MediaSearcherState.CLOSE;
	
	private StreamsManagerConfiguration config = null;
	
	private StoreManager storeManager;

	private Jedis subscriberJedis;
	
	private DyscoRequestHandler dyscoRequestHandler;
	private DyscoRequestReceiver dyscoRequestReceiver;
	private TrendingSearchHandler trendingSearchHandler;
	private CustomSearchHandler customSearchHandler;
	
	private String redisHost;
	private String solrHost;
	private String solrService;
	private String dyscoCollection;
	private String newsfeedHost;
	private String newsfeedDatabase;
	private String newsfeedCollection;
	
	private Map<String, Stream> streams = null;
	
	private Queue<Dysco> requests = new LinkedList<Dysco>();
	
	
	public MediaSearcher(StreamsManagerConfiguration config) throws StreamException{
		if (config == null) {
			throw new StreamException("Manager's configuration must be specified");
		}

		this.config = config;
		this.redisHost = config.getParameter(MediaSearcher.REDIS_HOST);
		this.solrHost = config.getParameter(MediaSearcher.SOLR_HOST);
		this.solrService = config.getParameter(MediaSearcher.SOLR_SERVICE);
		this.dyscoCollection = config.getParameter(MediaSearcher.DYSCO_COLLECTION);
		
		
		//Set up the Streams
		initStreams();

		Runtime.getRuntime().addShutdownHook(new Shutdown(this));
	}
	
	/**
	 * Opens Manager by starting the auxiliary modules and setting up
	 * the database for reading/storing
	 * @throws StreamException
	 */
	public synchronized void open() throws StreamException {
		if (state == MediaSearcherState.OPEN) {
			return;
		}
		state = MediaSearcherState.OPEN;
		
		storeManager = new StoreManager(config);
		storeManager.start();	
		logger.info("Store Manager is ready to store.");
		<commit_msg>if mongo is not working media searcher continues working, only when solr is closed it stops<commit_after>package eu.socialsensor.sfc.streams.management;


import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;

import javax.xml.parsers.ParserConfigurationException;

import org.apache.log4j.Logger;
import org.xml.sax.SAXException;

import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig;
import redis.clients.jedis.JedisPubSub;

import eu.socialsensor.framework.client.search.solr.SolrDyscoHandler;
import eu.socialsensor.framework.common.domain.Feed;
import eu.socialsensor.framework.common.domain.dysco.Dysco;
import eu.socialsensor.framework.common.domain.dysco.Dysco.DyscoType;
import eu.socialsensor.framework.common.domain.dysco.Message;
import eu.socialsensor.framework.common.domain.dysco.Message.Action;
import eu.socialsensor.framework.streams.Stream;
import eu.socialsensor.framework.streams.StreamConfiguration;
import eu.socialsensor.framework.streams.StreamException;
import eu.socialsensor.sfc.builder.FeedsCreator;
import eu.socialsensor.sfc.builder.input.DataInputType;
import eu.socialsensor.sfc.streams.StreamsManagerConfiguration;
import eu.socialsensor.sfc.streams.monitors.StreamsMonitor;

public class MediaSearcher {
	private static String REDIS_HOST = "redis.host";
	private static String SOLR_HOST = "solr.hostname";
	private static String SOLR_SERVICE = "solr.service";
	private static String DYSCO_COLLECTION = "dyscos.collection";
	
	public final Logger logger = Logger.getLogger(StreamsManager.class);
	
	enum MediaSearcherState {
		OPEN, CLOSE
	}
	private MediaSearcherState state = MediaSearcherState.CLOSE;
	
	private StreamsManagerConfiguration config = null;
	
	private StoreManager storeManager;

	private Jedis subscriberJedis;
	
	private DyscoRequestHandler dyscoRequestHandler;
	private DyscoRequestReceiver dyscoRequestReceiver;
	private TrendingSearchHandler trendingSearchHandler;
	private CustomSearchHandler customSearchHandler;
	
	private String redisHost;
	private String solrHost;
	private String solrService;
	private String dyscoCollection;
	
	private Map<String, Stream> streams = null;
	
	private Queue<Dysco> requests = new LinkedList<Dysco>();
	
	
	public MediaSearcher(StreamsManagerConfiguration config) throws StreamException{
		if (config == null) {
			throw new StreamException("Manager's configuration must be specified");
		}

		this.config = config;
		this.redisHost = config.getParameter(MediaSearcher.REDIS_HOST);
		this.solrHost = config.getParameter(MediaSearcher.SOLR_HOST);
		this.solrService = config.getParameter(MediaSearcher.SOLR_SERVICE);
		this.dyscoCollection = config.getParameter(MediaSearcher.DYSCO_COLLECTION);
		
		
		//Set up the Streams
		initStreams();

		Runtime.getRuntime().addShutdownHook(new Shutdown(this));
	}
	
	/**
	 * Opens Manager by starting the auxiliary modules and setting up
	 * the database for reading/storing
	 * @throws StreamException
	 */
	public synchronized void open() throws StreamException {
		if (state == MediaSearcherState.OPEN) {
			return;
		}
		state = MediaSearcherState.OPEN;
		
		storeManager = new StoreManager(config);
		
		if(!storeManager.getWorkingDataBases().get("Solr")){
			System.out.println("Apache solr is not working - Close Media Searcher");
			state = MediaSearcherState.CLOSE;
			return;
		}
		
		storeManager.start();	
		logger.info("Store Manager is ready to store.");
		
{'fraction_non_alphanumeric': 0.08848931049129265, 'fraction_numerical': 0.0002831657935721365, 'mean_word_length': 5.626641651031895, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7661709', 'n_tokens_mistral': 2307, 'n_tokens_neox': 2143, 'n_words': 452}