<commit_before>package de.zib.gndms.infra.configlet;

import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import de.zib.gndms.c3resource.C3ResourceReader;
import de.zib.gndms.c3resource.jaxb.Site;
import de.zib.gndms.c3resource.jaxb.Workspace;
import de.zib.gndms.kit.config.MandatoryOptionMissingException;
import de.zib.gndms.kit.configlet.RegularlyRunnableConfiglet;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.message.BasicHeader;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

import java.io.IOException;
import java.io.InputStream;
import java.io.Serializable;
import java.net.URL;
import java.util.*;


/**
 * ThingAMagic.
 *
 * @author <NAME><<EMAIL>>
 * @version $Id$
 *
 *          User: stepn Date: 06.11.2008 Time: 18:33:18
 */
@SuppressWarnings({ "ClassNamingConvention", "ReturnOfCollectionOrArrayField" })
public class C3MDSConfiglet extends RegularlyRunnableConfiglet {
	private String mdsUrl;
	private String requiredPrefix;

	private C3Catalog catalog;

	@Override
	protected synchronized void threadInit() {
		super.threadInit();
		configProperties();
	}



	@Override
	public synchronized void update(@NotNull final Serializable data) {
		super.update(data);    // Overridden method
		configProperties();
	}


	private synchronized void configProperties() {
		try {
			mdsUrl = getMapConfig().getOption("mdsUrl");
			requiredPrefix = getMapConfig().getOption("requiredPrefix", "");
		}
		catch ( MandatoryOptionMissingException e) {
			getLog().warn(e);
		}
	}

	@Override
	protected synchronized void threadRun() {
		try {
			getLog().info("Refreshing C3MDSCatalog...");
			final C3ResourceReader reader = new C3ResourceReader();
			final String curRequiredPrefix = getRequiredPrefix();
			final InputStream inputStream = openMdsInputStream();
			C3Catalog newCatalog = null;
			// pointless since MDS includes dynamically generated timestamps...
			//final MD5InputStream checkedStream = new MD5InputStream(inputStream);
			try {
				final Iterator<Site> sites = reader.readXmlSites(curRequiredPrefix, inputStream);
				newCatalog = new C3Catalog(curRequiredPrefix, sites);
			}
			finally {
				try {
					inputStream.close();
					if (newCatalog == null)
						getLog().warn("No new C3MDSCatalog was created (unknown reason)");
					setCatalog(newCatalog);
				}
				catch (IOException e)
					{ getLog().warn("Error closing MDS stream; new catalog *not* set"); }
			}
			getLog().debug("Finished Refreshing C3MDSCatalog");
		}
		catch (RuntimeException e) {
			getLog().warn(e);
		}
		catch (IOException e) {
			getLog().warn(e);
		}
	}


	private InputStream openMdsInputStream() throws IOException {
		final String urlStr = getMdsUrl();
		final URL url = new URL(urlStr);
		if (url.getProtocol().startsWith("http")) {
			getLog().debug("Loading C3MDSCatalog via http core...");
			// if http use http client from apache commons
			final HttpClient client = new DefaultHttpClient();
			final HttpGet get = new HttpGet(urlStr);
			get.addHeader(new BasicHeader("Pragma", "no-cache"));
			get.addHeader(new BasicHeader("Cache-Control",
			                              "private, no-store, no-cache, must-revalidate, max-age=0"));
			final HttpResponse resp = client.execute(get);
			return resp.getEntity().getContent();
		}
		else {
			getLog().debug("Loading C3MDSCatalog via java.net.URL.openStream...");
			// defer to java-built in url handling otherwise
			return url.openStream();
		}
	}


	public synchronized C3Catalog getCatalog() {
		while (getCatalog() == null)
			try {
				wait();<commit_msg>Minor fix in C3MDSConfiglet::getCatalog<commit_after>package de.zib.gndms.infra.configlet;

import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import de.zib.gndms.c3resource.C3ResourceReader;
import de.zib.gndms.c3resource.jaxb.Site;
import de.zib.gndms.c3resource.jaxb.Workspace;
import de.zib.gndms.kit.config.MandatoryOptionMissingException;
import de.zib.gndms.kit.configlet.RegularlyRunnableConfiglet;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.message.BasicHeader;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

import java.io.IOException;
import java.io.InputStream;
import java.io.Serializable;
import java.net.URL;
import java.util.*;


/**
 * ThingAMagic.
 *
 * @author <NAME><<EMAIL>>
 * @version $Id$
 *
 *          User: stepn Date: 06.11.2008 Time: 18:33:18
 */
@SuppressWarnings({ "ClassNamingConvention", "ReturnOfCollectionOrArrayField" })
public class C3MDSConfiglet extends RegularlyRunnableConfiglet {
	private String mdsUrl;
	private String requiredPrefix;

	private C3Catalog catalog;

	@Override
	protected synchronized void threadInit() {
		super.threadInit();
		configProperties();
	}



	@Override
	public synchronized void update(@NotNull final Serializable data) {
		super.update(data);    // Overridden method
		configProperties();
	}


	private synchronized void configProperties() {
		try {
			mdsUrl = getMapConfig().getOption("mdsUrl");
			requiredPrefix = getMapConfig().getOption("requiredPrefix", "");
		}
		catch ( MandatoryOptionMissingException e) {
			getLog().warn(e);
		}
	}

	@Override
	protected synchronized void threadRun() {
		try {
			getLog().info("Refreshing C3MDSCatalog...");
			final C3ResourceReader reader = new C3ResourceReader();
			final String curRequiredPrefix = getRequiredPrefix();
			final InputStream inputStream = openMdsInputStream();
			C3Catalog newCatalog = null;
			// pointless since MDS includes dynamically generated timestamps...
			//final MD5InputStream checkedStream = new MD5InputStream(inputStream);
			try {
				final Iterator<Site> sites = reader.readXmlSites(curRequiredPrefix, inputStream);
				newCatalog = new C3Catalog(curRequiredPrefix, sites);
			}
			finally {
				try {
					inputStream.close();
					if (newCatalog == null)
						getLog().warn("No new C3MDSCatalog was created (unknown reason)");
					setCatalog(newCatalog);
				}
				catch (IOException e)
					{ getLog().warn("Error closing MDS stream; new catalog *not* set"); }
			}
			getLog().debug("Finished Refreshing C3MDSCatalog");
		}
		catch (RuntimeException e) {
			getLog().warn(e);
		}
		catch (IOException e) {
			getLog().warn(e);
		}
	}


	private InputStream openMdsInputStream() throws IOException {
		final String urlStr = getMdsUrl();
		final URL url = new URL(urlStr);
		if (url.getProtocol().startsWith("http")) {
			getLog().debug("Loading C3MDSCatalog via http core...");
			// if http use http client from apache commons
			final HttpClient client = new DefaultHttpClient();
			final HttpGet get = new HttpGet(urlStr);
			get.addHeader(new BasicHeader("Pragma", "no-cache"));
			get.addHeader(new BasicHeader("Cache-Control",
			                              "private, no-store, no-cache, must-revalidate, max-age=0"));
			final HttpResponse resp = client.execute(get);
			return resp.getEntity().getContent();
		}
		else {
			getLog().debug("Loading C3MDSCatalog via java.net.URL.openStream...");
			// defer to java-built in url handling otherwise
			return url.openStream();
		}
	}


	public synchronized C3Catalog getCatalog() {
		while ( catalog == null )
			try {
				wait();
{'fraction_non_alphanumeric': 0.12109945558358784, 'fraction_numerical': 0.008896560881689018, 'mean_word_length': 4.19448275862069, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3137036', 'n_tokens_mistral': 2595, 'n_tokens_neox': 2326, 'n_words': 522}