<commit_before>package src;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class CommunityInfoReader extends HTMLReader {
	private final static String urlDir = "https://ideascale.com/index/";
	private StatisticReader statsReader = null;
	
	public CommunityInfoReader() {
		super();
		prepareUserAgent();
		statsReader = new StatisticReader();
	}
	
	public List<HashMap<String,String>> getCommunitiesInfo() 
	throws Exception
	{
		List<HashMap<String,String>> comunities = new ArrayList<HashMap<String,String>>();
		List<String> letterDir = new ArrayList<String>(
					Arrays.asList("a","b","c","d","e","f","g","h","i","j","k",
							      "l","m","n","o","p","q","r","s","t","u","v",
							      "w","x","y","z"));
		for (String letter : letterDir) {
			System.out.println("Starting with communities of cat: " + letter.toUpperCase() + ".");
			String content = getUrlContent(urlDir+letter+".html");
	        Document doc = Jsoup.parse(content);	        
	        Elements liElements = doc.getElementsByTag("li");
	        for (Element li : liElements)
	        	if (li.children().size() > 0) {
	        		String link = li.child(0).attr("href");
	        		String name = li.text();
	        		if (link.contains("http://") && !inBlackList(link)) {
	        			HashMap<String,String> comunity = new HashMap<String,String>();
	        			comunity.put("name", name);
	        			comunity.put("url", link);
	        			System.out.println("Community: " + name);
	        			HashMap<String,String> comStatistics = statsReader.getCommunityStatistic(link);
	        			comunity.put("ideas", comStatistics.get("ideas"));
	        			comunity.put("members", comStatistics.get("members"));
	        			comunity.put("votes", comStatistics.get("votes"));
	        			comunity.put("comments", comStatistics.get("comments"));
	        			/*System.out.println(" - members: " + comStatistics.get("members") <commit_msg>Add a method to replace the thousand-symbol 'K' with 000<commit_after>package src;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class CommunityInfoReader extends HTMLReader {
	private final static String urlDir = "https://ideascale.com/index/";
	private StatisticReader statsReader = null;
	
	public CommunityInfoReader() {
		super();
		prepareUserAgent();
		statsReader = new StatisticReader();
	}
	
	public List<HashMap<String,String>> getCommunitiesInfo() 
	throws Exception
	{
		List<HashMap<String,String>> comunities = new ArrayList<HashMap<String,String>>();
		List<String> letterDir = new ArrayList<String>(
					Arrays.asList("a","b","c","d","e","f","g","h","i","j","k",
							      "l","m","n","o","p","q","r","s","t","u","v",
							      "w","x","y","z"));
		for (String letter : letterDir) {
			System.out.println("Starting with communities of cat: " + letter.toUpperCase() + ".");
			String content = getUrlContent(urlDir+letter+".html");
	        Document doc = Jsoup.parse(content);	        
	        Elements liElements = doc.getElementsByTag("li");
	        for (Element li : liElements)
	        	if (li.children().size() > 0) {
	        		String link = li.child(0).attr("href");
	        		String name = li.text();
	        		if (link.contains("http://") && !inBlackList(link)) {
	        			HashMap<String,String> comunity = new HashMap<String,String>();
	        			comunity.put("name", name.replace("\"",""));
	        			comunity.put("url", link);
	        			System.out.println("Community: " + name);
	        			HashMap<String,String> comStatistics = statsReader.getCommunityStatistic(link);
	        			comunity.put("ideas", comStatistics.get("ideas"));
	        			comunity.put("members", comStatistics.get("members"));
	        			comunity.put("votes", comStatistics.get("votes"));
	        			comunity.put("comments", comStatistics.get("comments"));
	        			/*System.out.println(" - members: " + comStatistics.get("members") 
{'fraction_non_alphanumeric': 0.1714622641509434, 'fraction_numerical': 0.0016509433962264152, 'mean_word_length': 3.4689146469968386, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 27, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3193992', 'n_tokens_mistral': 1414, 'n_tokens_neox': 1293, 'n_words': 239}