<commit_before>/*
 * Copyright 2014 hbz NRW (http://www.hbz-nrw.de/)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */
package helper;

import java.io.File;
import java.io.FileInputStream;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.KeyStore;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Comparator;

import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManagerFactory;

import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.api.client.config.ClientConfig;
import com.sun.jersey.api.client.config.DefaultClientConfig;
import com.sun.jersey.api.client.filter.HTTPDigestAuthFilter;
import com.sun.jersey.client.urlconnection.HTTPSProperties;

import play.Play;
import models.Gatherconf;
import models.Globals;

/**
 * @author <NAME>
 *
 */
public class Heritrix {

    class HeritrixWebclient {
	public Client createWebclient() {
	    String keystoreLoc = Play.application().configuration()
		    .getString("regal-api.heritrix.keystoreloc");
	    String keystorePass = Play.application().configuration()
		    .getString("regal-api.heritrix.keystorepwd");
	    String heritrixUser = Play.application().configuration()
		    .getString("regal-api.heritrix.user");
	    String heritrixPwd = Play.application().configuration()
		    .getString("regal-api.heritrix.pwd");

	    Client webclient = null;
	    ClientConfig cc = new DefaultClientConfig();
	    // cc.getProperties()
	    // .put(ClientConfig.PROPERTY_FOLLOW_REDIRECTS, true);
	    cc.getFeatures().put(ClientConfig.FEATURE_DISABLE_XML_SECURITY,
		    true);

	    cc.getProperties().put(
		    HTTPSProperties.PROPERTY_HTTPS_PROPERTIES,
		    new HTTPSProperties(null, initSsl(cc, keystoreLoc,
			    keystorePass)));
	    webclient = Client.create(cc);
	    webclient.addFilter(new HTTPDigestAuthFilter(heritrixUser,
		    heritrixPwd));
	    return webclient;
	}

	private SSLContext initSsl(ClientConfig cc, String keystorelocation,
		String keystorepasswd) {
	    try {
		SSLContext ctx = SSLContext.getInstance("SSL");
		KeyStore trustStore;
		trustStore = KeyStore.getInstance("JKS");
		trustStore.load(new FileInputStream(keystorelocation),
			keystorepasswd.toCharArray());<commit_msg>fix: potential resource leak<commit_after>/*
 * Copyright 2014 hbz NRW (http://www.hbz-nrw.de/)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */
package helper;

import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.KeyStore;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Comparator;

import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManagerFactory;

import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.api.client.config.ClientConfig;
import com.sun.jersey.api.client.config.DefaultClientConfig;
import com.sun.jersey.api.client.filter.HTTPDigestAuthFilter;
import com.sun.jersey.client.urlconnection.HTTPSProperties;

import play.Play;
import models.Gatherconf;
import models.Globals;

/**
 * @author <NAME>
 *
 */
public class Heritrix {

    class HeritrixWebclient {
	public Client createWebclient() {
	    String keystoreLoc = Play.application().configuration()
		    .getString("regal-api.heritrix.keystoreloc");
	    String keystorePass = Play.application().configuration()
		    .getString("regal-api.heritrix.keystorepwd");
	    String heritrixUser = Play.application().configuration()
		    .getString("regal-api.heritrix.user");
	    String heritrixPwd = Play.application().configuration()
		    .getString("regal-api.heritrix.pwd");

	    Client webclient = null;
	    ClientConfig cc = new DefaultClientConfig();
	    // cc.getProperties()
	    // .put(ClientConfig.PROPERTY_FOLLOW_REDIRECTS, true);
	    cc.getFeatures().put(ClientConfig.FEATURE_DISABLE_XML_SECURITY,
		    true);
	    cc.getProperties().put(
		    HTTPSProperties.PROPERTY_HTTPS_PROPERTIES,
		    new HTTPSProperties(null, initSsl(cc, keystoreLoc,
			    keystorePass)));
	    webclient = Client.create(cc);
	    webclient.addFilter(new HTTPDigestAuthFilter(heritrixUser,
		    heritrixPwd));
	    return webclient;
	}

	private SSLContext initSsl(ClientConfig cc, String keystorelocation,
		String keystorepasswd) {
	    try (InputStream keystoreInput = new FileInputStream(
		    keystorelocation)) {
		SSLContext ctx = SSLContext.getInstance("SSL");
		KeyStore trustStore;
		trustStore = KeyStore.getInstance("JKS");
		trustStore.load(keystoreInput, keystorepasswd.toCharArray());
{'fraction_non_alphanumeric': 0.10773668137088564, 'fraction_numerical': 0.0027146250424160165, 'mean_word_length': 4.258697591436218, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6852933', 'n_tokens_mistral': 1878, 'n_tokens_neox': 1766, 'n_words': 410}