<commit_before>package focusedCrawler.link.frontier;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import focusedCrawler.util.persistence.PersistentHashtable;

public class AddSeeds {
	
	private static final Logger logger = LoggerFactory.getLogger(AddSeeds.class);

	public static void main(String[] args) {
		try {
            String linkConfigFile = args[0] + "/link_storage/link_storage.cfg";
            String seedFile = args[1];
            String dataPath = args[2];
            
			focusedCrawler.util.ParameterFile config = new focusedCrawler.util.ParameterFile(linkConfigFile);
//			focusedCrawler.util.ParameterFile seedConfig = new focusedCrawler.util.ParameterFile(seedFile);
			
			String dir = dataPath + "/" + config.getParam("LINK_DIRECTORY");
			
			PersistentHashtable urls = new PersistentHashtable(dir,1000);
			FrontierTargetRepositoryBaseline frontier = new FrontierTargetRepositoryBaseline(urls,10000);
			int count = 0;
            /*
			if(args.length > 1){
				BufferedReader input = new BufferedReader(new FileReader(args[1]));
				for (String line1 = input.readLine(); line1 != null; line1 = input.readLine()) {
					LinkRelevance linkRel = new LinkRelevance(new URL(line1), 299);
					frontier.insert(linkRel);
					count++;
				}
			}else{
				String[] seeds = config.getParam("SEEDS"," ");
				for (int i = 0; i < seeds.length; i++) {
					urls.put(seeds[i], "299");
//					LinkRelevance linkRel = new LinkRelevance(new URL(seeds[i]), 299);
//					boolean added = frontier.insert(linkRel);
//					System.out.println(added);
					count++;
				}
			}*/
            //String[] seeds = seedConfig.getParam("SEEDS"," ");
            String[] seeds = focusedCrawler.util.ParameterFile.getSeeds(seedFile);
            for (int i = 0; i < seeds.length; i++) {
              urls.put(seeds[i], "299");
              count++;
            }
			logger.info("Number of seeds:" + count);
			frontier.close();
			
		} catch (Exception e) {
			logger.error("Problem while adding seeds. ", e);<commit_msg>Add seeds only when they are not already stored<commit_after>package focusedCrawler.link.frontier;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import focusedCrawler.util.persistence.PersistentHashtable;

public class AddSeeds {
	
	private static final Logger logger = LoggerFactory.getLogger(AddSeeds.class);

	public static void main(String[] args) {
		try {
            String linkConfigFile = args[0] + "/link_storage/link_storage.cfg";
            String seedFile = args[1];
            String dataPath = args[2];
            
			focusedCrawler.util.ParameterFile config = new focusedCrawler.util.ParameterFile(linkConfigFile);
			
			String dir = dataPath + "/" + config.getParam("LINK_DIRECTORY");
			
			PersistentHashtable urls = new PersistentHashtable(dir,1000);
			FrontierTargetRepositoryBaseline frontier = new FrontierTargetRepositoryBaseline(urls,10000);
			int count = 0;
            /*
			if(args.length > 1){
				BufferedReader input = new BufferedReader(new FileReader(args[1]));
				for (String line1 = input.readLine(); line1 != null; line1 = input.readLine()) {
					LinkRelevance linkRel = new LinkRelevance(new URL(line1), 299);
					frontier.insert(linkRel);
					count++;
				}
			}else{
				String[] seeds = config.getParam("SEEDS"," ");
				for (int i = 0; i < seeds.length; i++) {
					urls.put(seeds[i], "299");
//					LinkRelevance linkRel = new LinkRelevance(new URL(seeds[i]), 299);
//					boolean added = frontier.insert(linkRel);
//					System.out.println(added);
					count++;
				}
			}*/
            //String[] seeds = seedConfig.getParam("SEEDS"," ");
            String[] seeds = focusedCrawler.util.ParameterFile.getSeeds(seedFile);
            for (int i = 0; i < seeds.length; i++) {
              if(urls.get(seeds[i]) == null) {
                  urls.put(seeds[i], "299");
                  count++;
              }
            }
			logger.info("Number of seeds added:" + count);
			frontier.close();
			
		} catch (Exception e) {
			logger.error("Problem while adding seeds. ", e);
{'fraction_non_alphanumeric': 0.13223748773307165, 'fraction_numerical': 0.01717369970559372, 'mean_word_length': 2.973684210526316, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2557179', 'n_tokens_mistral': 1466, 'n_tokens_neox': 1238, 'n_words': 270}