<commit_before>/**
 * Main class for manifestParsing, it takes the URL, parse its content.
 */

package downloader.manifest;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;

import downloader.utils.Utils;

public class ManifestParser {

	private Manifest manifestFile;
	private ArrayList<UrlLine> segments;

	/**
	 * ManifestParser constructor, it takes Manifest file as an object to extract
	 * the content
	 * 
	 * @param Object manifestFile manifestFile object
	 * 
	 */
	
	public ManifestParser(Manifest manifestFile) {
		this.manifestFile = manifestFile;
		segments = new ArrayList<UrlLine>();
	}

	/**
	 * getURLcontentType function, it takes Manifest url and extracts the
	 * content type
	 * 
	 * @param String manifestUrl manifest string url
	 * 
	 * @return string content type
	 * 
	 * @throws IOException
	 * 
	 */
	
	private String getURLcontentType(String manifestUrl) throws IOException {
		URL url = new URL(manifestUrl);
		HttpURLConnection connection = (HttpURLConnection) url.openConnection();
		connection.setRequestMethod("HEAD");
		connection.connect();
		return connection.getContentType();
	}

	/**
	 * isValidManifestUrl function, check if the manifest url is valid
	 * 
	 * @param String url, manifest string url
	 * 
	 * @return boolean true if url is valid manifest Url, false otherwise
	 * 
	 * @throws IOException
	 * 
	 */
	
	private boolean isValidManifestUrl(String url) throws IOException {
		if (url.endsWith(".segments") || getURLcontentType(url).equals("text/segments-manifest")) {
			return true;
		}
		return false;
	}

	/**
	 * getSegments function, extracts the segments from manifest file
	 * 
	 * @param
	 * 
	 * @return ArrayList, returns array list of segments and manifest urls
	 * inside manifest file
	 * 
	 * @throws IOException
	 * 
	 */
	
	public ArrayList<UrlLine> getSegments() throws IOException {
		ArrayList<String> manifestLines = extractManifestLines();
		boolean isAlternative = false;<commit_msg>Added invalideUrlException to use in case the url is not valid<commit_after>/**
 * Main class for manifestParsing, it takes the URL, parse its content.
 */

package downloader.manifest;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;

import downloader.exceptions.invalidUrlException;
import downloader.utils.Utils;

public class ManifestParser {

	private Manifest manifestFile;
	private ArrayList<UrlLine> segments;

	/**
	 * ManifestParser constructor, it takes Manifest file as an object to extract
	 * the content
	 * 
	 * @param Object manifestFile manifestFile object
	 * 
	 */
	
	public ManifestParser(Manifest manifestFile) {
		this.manifestFile = manifestFile;
		segments = new ArrayList<UrlLine>();
	}

	/**
	 * getURLcontentType function, it takes Manifest url and extracts the
	 * content type
	 * 
	 * @param String manifestUrl manifest string url
	 * 
	 * @return string content type
	 * 
	 * @throws IOException
	 * @throws invalidUrlException 
	 * 
	 */
	
	private String getURLcontentType(String manifestUrl) throws IOException, invalidUrlException {
		if(Utils.isUrl(manifestUrl)) {
		URL url = new URL(manifestUrl);
		HttpURLConnection connection = (HttpURLConnection) url.openConnection();
		connection.setRequestMethod("HEAD");
		connection.connect();
		return connection.getContentType();
		} else {
			throw new invalidUrlException("The URL provided is not valid URL");
		}
	}

	/**
	 * isValidManifestUrl function, check if the manifest url is valid
	 * 
	 * @param String url, manifest string url
	 * 
	 * @return boolean true if url is valid manifest Url, false otherwise
	 * 
	 * @throws IOException
	 * @throws invalidUrlException 
	 * 
	 */
	
	private boolean isValidManifestUrl(String url) throws IOException, invalidUrlException {
		if (url.endsWith(".segments") || getURLcontentType(url).equals("text/segments-manifest")) {
			return true;
		}
		return false;
	}

	/**
	 * getSegments function, extracts the segments from manifest file
	 * 
	 * @param
	 * 
	 * @return ArrayList, returns array list of segments and manifest urls
	 * inside manifest file
	 * 
	 * @throws IOException
	 * @throws invalidUrlException 
	 * 
	 */
	
	public ArrayList<UrlLine> getSegments() throws IOException, invalidUrlException {
		ArrayList<String> manifestLines = extractManifestLines();
		boolean isAlternative = false;
{'fraction_non_alphanumeric': 0.08950749464668094, 'fraction_numerical': 0.0, 'mean_word_length': 4.138613861386139, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3226985', 'n_tokens_mistral': 1393, 'n_tokens_neox': 1220, 'n_words': 399}