<commit_before>package br.com.ivy.service.scan;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.regex.Pattern;
import java.util.regex.Matcher;

public class HtmlAnalyser {
	
	private int listPosition;
	
	private Matcher matcher;
	
	private List<String> links;
	
	private Set<String> sampling;
	
	private static final int tryNumber = 5;
	
	private final Pattern linkRegex = Pattern.compile("href=\"(.*?)\"");
	
	public HtmlAnalyser(){
		links 	 = new ArrayList<String>();
		sampling = new HashSet<String>();
	}

	public Set<String> getSampling(){
		return this.sampling;
	}

	public String linkFormat(String link, String host){
		
		if(link.startsWith("#") || link.startsWith("\\")){
			link = null;
		}else if(link.startsWith("/")){
			link = String.format("http://%s%s", host, link);
		}else if(!link.contains(host)){
			link = null;
		}
		return link;
	}

	public void linkChecker(URL host) throws IOException{

		String content = getContent(host);

		String[] lines = content.split("\n");
		
		String link = null;

		for (int i=0; i < lines.length; i++) {
			if(sampling.size() >= tryNumber) break;
			
			if(!lines[i].contains("<a")) continue;
			
			this.matcher = this.linkRegex.matcher(lines[i]);
			
			if(matcher.find()) {
				link = linkFormat(matcher.group(1), host.getHost());
				if(link != null){
					this.links.add(link);
					if(link.contains("?")) this.sampling.add(link);
				}
			}
		}
		
		if(sampling.size() < tryNumber && this.links.size() >= this.listPosition){
			linkChecker(new URL(this.links.get(this.listPosition++)));
		}
	}
	
	public String getContent(URL host){
		try{
			HttpURLConnection connection = (HttpURLConnection) host.openConnection();
	
			HttpURLConnection.setFollowRedirects(false);
			connection.setRequestMethod("GET");
	
			BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream(), "UTF-8"));
	
			String line;
			StringBuilder content = new StringBuilder();
	
			while ((line = reader.readLine()) != null) content.append(String.format("%s\n",line));
	
			reader.close();
			connection.disconnect();
	
			return content.toString();
		}catch(Exception e){ return null; }
	}
}
<commit_msg>Ivy - diversidade de rotas de busca<commit_after>package br.com.ivy.service.scan;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.regex.Pattern;
import java.util.regex.Matcher;

public class HtmlAnalyser {
	
	private Matcher matcher;
	
	private List<String> links;
	
	private Set<String> sampling;
	
	private static final int tryNumber = 5;
	
	private final Pattern linkRegex = Pattern.compile("href=\"(.*?)\"");
	
	public HtmlAnalyser(){
		links 	 = new ArrayList<String>();
		sampling = new HashSet<String>();
	}

	public Set<String> getSampling(){
		return this.sampling;
	}

	public void linkChecker(URL host) throws IOException{

		String content = getContent(host);

		String[] lines = content.split("\n");

		for (int i=0; i < lines.length; i++) {
			if(sampling.size() >= tryNumber) break;
			
			if(!lines[i].contains("<a")) continue;
			
			this.matcher = this.linkRegex.matcher(lines[i]);
			
			if(matcher.find()) {
				String link = linkFormat(matcher.group(1), host.getHost());
				if(link != null){
					this.links.add(link);
					if(link.contains("?")) this.sampling.add(link);
				}
			}
		}
		
		if(sampling.size() < tryNumber && this.links.size() > 0){
			Collections.shuffle(links);
			String link = this.links.get(0);
			this.links.remove(0);
			linkChecker(new URL(link));
		}
	}
	
	public String getContent(URL host){
		try{
			HttpURLConnection connection = (HttpURLConnection) host.openConnection();
	
			HttpURLConnection.setFollowRedirects(false);
			connection.setRequestMethod("GET");
	
			BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream(), "UTF-8"));
	
			String line;
			StringBuilder content = new StringBuilder();
	
			while ((line = reader.readLine()) != null) content.append(String.format("%s\n",line));
	
			reader.close();
			connection.disconnect();
	
			return content.toString();
		}catch(Exception e){ return null; }
	}
	
	public String linkFormat(String link, String host){
		
		if(link.startsWith("#") || link.startsWith("\\")){
			link = null;
		}else if(link.startsWith("/")){
			link = String.format("http://%s%s", host, link);
		}else if(!link.contains(host)){
			link = null;
		}
		return link;
	}
}

{'fraction_non_alphanumeric': 0.15389395194697597, 'fraction_numerical': 0.0022783761391880694, 'mean_word_length': 4.067156348373557, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '932832', 'n_tokens_mistral': 1765, 'n_tokens_neox': 1526, 'n_words': 335}