<commit_before>/**
 * 
 */
package de.csw.cl.importer.algorithm;

import static util.XMLUtil.NS_XCL2;

import java.io.File;
import java.io.FilenameFilter;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Queue;
import java.util.SortedSet;
import java.util.Stack;
import java.util.TreeSet;

import org.jdom2.Comment;
import org.jdom2.Content;
import org.jdom2.Document;
import org.jdom2.Element;
import org.jdom2.Parent;

import util.XMLUtil;
import de.csw.cl.importer.model.ConflictingTitlingException;
import de.csw.cl.importer.model.Corpus;
import de.csw.cl.importer.model.Includes;
import de.csw.cl.importer.model.XMLCatalog;

/**
 * @author ralph
 *
 */
public class CLImportationAlgorithm {
	
	private enum ELEMENT_TYPE {Titling, Restrict, Import, include, other}
	
	private File baseDir;
	
	private File resultDir;
	private File includesDir;
	
	private Corpus corpus;
	private Includes includes;
	private XMLCatalog catalog;
	
	// not used for now
	private final Queue<Element> potentiallyPendingImports = new LinkedList<Element>();

	private Stack<String> importHistory;
	
	private Integer includeNumber = 0;
	
	/**
	 * Constructs a {@link CLImportationAlgorithm} object initialized with a given base file.
	 * If the base file resides in directory inputDir, the following file layout will be used:
	 * <ul>
	 * <li> All xcl files residing in inputDir will be considered the corpus.
	 * <li> The resulting xcl file with the importation closure will be saved to inputDir/../result.
	 * <li> Additional files for inclusion will be saved in inputDir/result/includes.
	 * </ul>
	 * @param inputFile
	 */
	public CLImportationAlgorithm(File corpusDirectory) {

		baseDir = corpusDirectory;
		resultDir = new File(baseDir.getParentFile(), "test-result");
		includesDir = new File(resultDir, "includes");

		includes = new Includes(includesDir);
		catalog = new XMLCatalog(new File(resultDir, "catalog.xml"));
	}
	
	/**
	 * Starts the importation process.
	 * @throws ConflictingTitlingException 
	 * @throws FolderCreationException 
	 */
	public void run() throws ConflictingTitlingException, FolderCreationException {
		
		try {
			loadCorpus();
		} catch (ConflictingTitlingException e) {
//			System.out.println("Error: The corpus includes two conflicting titlings with the same name: " + e.getName() + ". Aborting.");
			throw e;
		}
		
		processImports();
		
		//Iterable<Document> documentsInCorpus = corpus.getDocuments();
		
		if (corpus.size() > 0) {
	        if (!(resultDir.exists() || resultDir.mkdir())) {
	            throw new FolderCreationException("Error creating directory " + resultDir.getAbsolutePath());
	        }
	        for (Document document : corpus.getDocuments()) {
	            XMLUtil.writeXML(document, new File(resultDir, corpus.getOriginalFile(document).getName().replaceAll("myText", "resultText")));
	        }
	        catalog.write();
	        if (includes.size() > 0) {
	            if (!(includesDir.exists() || includesDir.mkdir())) {
	                throw new FolderCreationException("Error creating directory " + includesDir.getAbsolutePath());
	            }
	            includes.writeIncludes();
	        }
        }
	}
	
	/**
	 * Loads the corpus.
	 * @throws ConflictingTitlingException
	 */
	private void loadCorpus() throws ConflictingTitlingException {
		corpus = new Corpus();
		
		File[] files = baseDir.listFiles(new FilenameFilter() {<commit_msg>Old test result files are deleted prior to running a new test.<commit_after>/**
 * 
 */
package de.csw.cl.importer.algorithm;

import static util.XMLUtil.NS_XCL2;

import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.nio.file.FileSystem;
import java.nio.file.FileSystems;
import java.nio.file.FileVisitResult;
import java.nio.file.FileVisitor;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Queue;
import java.util.SortedSet;
import java.util.Stack;
import java.util.TreeSet;

import org.jdom2.Comment;
import org.jdom2.Content;
import org.jdom2.Document;
import org.jdom2.Element;
import org.jdom2.Parent;

import util.XMLUtil;
import de.csw.cl.importer.model.ConflictingTitlingException;
import de.csw.cl.importer.model.Corpus;
import de.csw.cl.importer.model.Includes;
import de.csw.cl.importer.model.XMLCatalog;

/**
 * @author ralph
 *
 */
public class CLImportationAlgorithm {
	
	private enum ELEMENT_TYPE {Titling, Restrict, Import, include, other}
	
	private File baseDir;
	
	private File resultDir;
	private File includesDir;
	
	private Corpus corpus;
	private Includes includes;
	private XMLCatalog catalog;
	
	// not used for now
	private final Queue<Element> potentiallyPendingImports = new LinkedList<Element>();

	private Stack<String> importHistory;
	
	private Integer includeNumber = 0;
	
	/**
	 * Constructs a {@link CLImportationAlgorithm} object initialized with a given base file.
	 * If the base file resides in directory inputDir, the following file layout will be used:
	 * <ul>
	 * <li> All xcl files residing in inputDir will be considered the corpus.
	 * <li> The resulting xcl file with the importation closure will be saved to inputDir/../result.
	 * <li> Additional files for inclusion will be saved in inputDir/result/includes.
	 * </ul>
	 * @param inputFile
	 */
	public CLImportationAlgorithm(File corpusDirectory) {

		baseDir = corpusDirectory;
		resultDir = new File(baseDir.getParentFile(), "test-result");
		includesDir = new File(resultDir, "includes");

		includes = new Includes(includesDir);
		catalog = new XMLCatalog(new File(resultDir, "catalog.xml"));
	}
	
	/**
	 * Starts the importation process.
	 * @throws ConflictingTitlingException 
	 * @throws FolderCreationException 
	 */
	public void run() throws ConflictingTitlingException, FolderCreationException {
		
		try {
			loadCorpus();
		} catch (ConflictingTitlingException e) {
//			System.out.println("Error: The corpus includes two conflicting titlings with the same name: " + e.getName() + ". Aborting.");
			throw e;
		}
		
		processImports();
		
		//Iterable<Document> documentsInCorpus = corpus.getDocuments();
		
		if (corpus.size() > 0) {
			try {
				Files.walkFileTree(resultDir.toPath(), new FileVisitor<Path>() {

					public FileVisitResult preVisitDirectory(Path dir,
							BasicFileAttributes attrs) throws IOException {
						return FileVisitResult.CONTINUE;
					}

					public FileVisitResult visitFile(Path file,
							BasicFileAttributes attrs) throws IOException {
						Files.delete(file);
						return FileVisitResult.CONTINUE;
					}

					public FileVisitResult visitFileFailed(Path file,
							IOException exc) throws IOException {
						return FileVisitResult.CONTINUE;
					}

					public FileVisitResult postVisitDirectory(Path dir,
							IOException exc) throws IOException {
						Files.delete(dir);
						return FileVisitResult.CONTINUE;
					}
				});
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
			
	        if (!resultDir.mkdir()) {
	            throw new FolderCreationException("Error creating directory " + resultDir.getAbsolutePath());
	        }
	        for (Document document : corpus.getDocuments()) {
	            XMLUtil.writeXML(document, new File(resultDir, corpus.getOriginalFile(document).getName().replaceAll("myText", "resultText")));
	        }
	        catalog.write();
	        if (includes.size() > 0) {
	            if (!(includesDir.exists() || includesDir.mkdir())) {
	                throw new FolderCreationException("Error creating directory " + includesDir.getAbsolutePath());
	            }
	            includes.writeIncludes();
	        }
        }
	}
	
	/**
	 * Loads the corpus.
	 * @throws ConflictingTitlingException
	 */
	private void loadCorpus() throws ConflictingTitlingException {
		corpus = new Corpus();
		
		File[] files = baseDir.listFiles(new FilenameFilter() {
{'fraction_non_alphanumeric': 0.10588235294117647, 'fraction_numerical': 0.002183141297756216, 'mean_word_length': 3.6799091940976165, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4559074', 'n_tokens_mistral': 2689, 'n_tokens_neox': 2364, 'n_words': 630}