<commit_before>/******************************************************************************
 *                                                                            *
 *                    Copyright 2016 Subterranean Security                    *
 *                                                                            *
 *  Licensed under the Apache License, Version 2.0 (the "License");           *
 *  you may not use this file except in compliance with the License.          *
 *  You may obtain a copy of the License at                                   *
 *                                                                            *
 *      http://www.apache.org/licenses/LICENSE-2.0                            *
 *                                                                            *
 *  Unless required by applicable law or agreed to in writing, software       *
 *  distributed under the License is distributed on an "AS IS" BASIS,         *
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  *
 *  See the License for the specific language governing permissions and       *
 *  limitations under the License.                                            *
 *                                                                            *
 *****************************************************************************/
package com.sandpolis.core.util;

import java.io.DataInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.net.URL;
import java.net.URLConnection;
import java.net.UnknownHostException;

import org.apache.commons.validator.routines.DomainValidator;
import org.apache.commons.validator.routines.InetAddressValidator;

/**
 * Network utilities.
 * 
 * @author cilki
 * @since 3.0.0
 */
public class NetUtil {

	/**
	 * The maximum number of bytes that can be downloaded with
	 * {@link #download(String)} or {@link #download(URL)}. This limit exists to
	 * prevent out of memory errors.
	 */
	private static final int DOWNLOAD_LIMIT = 512 * 1024 * 1024 * 1024;

	/**
	 * The number of milliseconds to wait before considering the port to be closed.
	 */
	public static final int PORTCHECK_TIMEOUT = 850;

	/**
	 * Download a file from the Internet. The entire file is loaded into memory, so
	 * only use with small files!
	 * 
	 * @param url The resource location
	 * @return A byte array containing the file
	 * @throws IOException
	 */
	public static byte[] download(String url) throws IOException {
		return download(new URL(url));
	}

	/**
	 * Download a file from the Internet. The entire file is loaded into memory, so
	 * only use with small files!
	 * 
	 * @param url The resource location
	 * @return A byte array containing the file
	 * @throws IOException
	 */
	public static byte[] download(URL url) throws IOException {
		if (url == null)
			throw new IllegalArgumentException();

		URLConnection con = url.openConnection();

		if (con.getContentLength() > DOWNLOAD_LIMIT)
			throw new IllegalArgumentException("File too large");

		try (DataInputStream in = new DataInputStream(con.getInputStream())) {
			return in.readAllBytes();
		}

	}

	/**
	 * Download a file from the Internet to a local file. This method has no size
	 * limits.<commit_msg>Use getContentLengthLong() instead of getContentLength()<commit_after>/******************************************************************************
 *                                                                            *
 *                    Copyright 2016 Subterranean Security                    *
 *                                                                            *
 *  Licensed under the Apache License, Version 2.0 (the "License");           *
 *  you may not use this file except in compliance with the License.          *
 *  You may obtain a copy of the License at                                   *
 *                                                                            *
 *      http://www.apache.org/licenses/LICENSE-2.0                            *
 *                                                                            *
 *  Unless required by applicable law or agreed to in writing, software       *
 *  distributed under the License is distributed on an "AS IS" BASIS,         *
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  *
 *  See the License for the specific language governing permissions and       *
 *  limitations under the License.                                            *
 *                                                                            *
 *****************************************************************************/
package com.sandpolis.core.util;

import java.io.DataInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.net.URL;
import java.net.URLConnection;
import java.net.UnknownHostException;

import org.apache.commons.validator.routines.DomainValidator;
import org.apache.commons.validator.routines.InetAddressValidator;

/**
 * Network utilities.
 * 
 * @author cilki
 * @since 3.0.0
 */
public final class NetUtil {
	private NetUtil() {
	}

	/**
	 * The maximum number of bytes that can be downloaded directly into memory with
	 * {@link #download(String)} or {@link #download(URL)}. This limit exists to
	 * prevent out of memory errors and is set to 512 MiB.
	 */
	private static final long DOWNLOAD_LIMIT = 536870912L;

	/**
	 * The number of milliseconds to wait before considering the port to be closed.
	 */
	public static final int PORTCHECK_TIMEOUT = 850;

	/**
	 * Download a file from the Internet. The entire file is loaded into memory, so
	 * only use with small files!
	 * 
	 * @param url The resource location
	 * @return A byte array containing the file
	 * @throws IOException
	 */
	public static byte[] download(String url) throws IOException {
		return download(new URL(url));
	}

	/**
	 * Download a file from the Internet. The entire file is loaded into memory, so
	 * only use with small files!
	 * 
	 * @param url The resource location
	 * @return A byte array containing the file
	 * @throws IOException
	 */
	public static byte[] download(URL url) throws IOException {
		if (url == null)
			throw new IllegalArgumentException();

		URLConnection con = url.openConnection();
		long size = con.getContentLengthLong();

		if (size > DOWNLOAD_LIMIT)
			throw new IllegalArgumentException("File too large: " + size + " bytes");

		try (DataInputStream in = new DataInputStream(con.getInputStream())) {
			return in.readAllBytes();
		}

	}

	/**
	 * Download a file from the Internet to a local file. This method has no size
	 * limits.
{'fraction_non_alphanumeric': 0.1196431183267515, 'fraction_numerical': 0.008044463946175223, 'mean_word_length': 2.0110083663584324, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 54, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2656583', 'n_tokens_mistral': 1653, 'n_tokens_neox': 1501, 'n_words': 599}